agora inbox for [email protected]help / color / mirror / Atom feed
[PATCH v28 10/11] Add regression tests for Incremental View Maintenance 259+ messages / 2 participants [nested] [flat]
* [PATCH v28 10/11] Add regression tests for Incremental View Maintenance @ 2021-03-10 02:11 Takuma Hoshiai <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Takuma Hoshiai @ 2021-03-10 02:11 UTC (permalink / raw) --- src/bin/pg_dump/pg_dump.h | 1 + .../regress/expected/incremental_matview.out | 1031 +++++++++++++++++ src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/incremental_matview.sql | 534 +++++++++ 4 files changed, 1567 insertions(+), 1 deletion(-) create mode 100644 src/test/regress/expected/incremental_matview.out create mode 100644 src/test/regress/sql/incremental_matview.sql diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index 119293a751..c3960a4192 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -322,6 +322,7 @@ typedef struct _tableInfo int numParents; /* number of (immediate) parent tables */ struct _tableInfo **parents; /* TableInfos of immediate parents */ + bool isivm; /* is incrementally maintainable materialized view? */ /* diff --git a/src/test/regress/expected/incremental_matview.out b/src/test/regress/expected/incremental_matview.out new file mode 100644 index 0000000000..b1e9424508 --- /dev/null +++ b/src/test/regress/expected/incremental_matview.out @@ -0,0 +1,1031 @@ +-- create a table to use as a basis for views and materialized views in various combinations +CREATE TABLE mv_base_a (i int, j int); +INSERT INTO mv_base_a VALUES + (1,10), + (2,20), + (3,30), + (4,40), + (5,50); +CREATE TABLE mv_base_b (i int, k int); +INSERT INTO mv_base_b VALUES + (1,101), + (2,102), + (3,103), + (4,104); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_1 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) WITH NO DATA; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +ERROR: materialized view "mv_ivm_1" has not been populated +HINT: Use the REFRESH MATERIALIZED VIEW command. +REFRESH MATERIALIZED VIEW mv_ivm_1; +NOTICE: could not create an index on materialized view "mv_ivm_1" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 10 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 +(4 rows) + +-- REFRESH WITH NO DATA +BEGIN; +CREATE FUNCTION dummy_ivm_trigger_func() RETURNS TRIGGER AS $$ + BEGIN + RETURN NULL; + END +$$ language plpgsql; +CREATE CONSTRAINT TRIGGER dummy_ivm_trigger AFTER INSERT +ON mv_base_a FROM mv_ivm_1 FOR EACH ROW +EXECUTE PROCEDURE dummy_ivm_trigger_func(); +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; + count +------- + 17 +(1 row) + +REFRESH MATERIALIZED VIEW mv_ivm_1 WITH NO DATA; +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; + count +------- + 1 +(1 row) + +ROLLBACK; +-- immediate maintenance +BEGIN; +INSERT INTO mv_base_b VALUES(5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 10 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 + 5 | 50 | 105 +(5 rows) + +UPDATE mv_base_a SET j = 0 WHERE i = 1; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 0 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 + 5 | 50 | 105 +(5 rows) + +DELETE FROM mv_base_b WHERE (i,k) = (5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 0 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 +(4 rows) + +ROLLBACK; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + i | j | k +---+----+----- + 1 | 10 | 101 + 2 | 20 | 102 + 3 | 30 | 103 + 4 | 40 | 104 +(4 rows) + +-- rename of IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_rename AS SELECT DISTINCT * FROM mv_base_a; +NOTICE: created index "mv_ivm_rename_index" on materialized view "mv_ivm_rename" +ALTER MATERIALIZED VIEW mv_ivm_rename RENAME COLUMN __ivm_count__ TO xxx; +ERROR: IVM column can not be renamed +DROP MATERIALIZED VIEW mv_ivm_rename; +-- unique index on IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_unique AS SELECT DISTINCT * FROM mv_base_a; +NOTICE: created index "mv_ivm_unique_index" on materialized view "mv_ivm_unique" +CREATE UNIQUE INDEX ON mv_ivm_unique(__ivm_count__); +ERROR: unique index creation on IVM columns is not supported +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__)); +ERROR: unique index creation on IVM columns is not supported +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__ + 1)); +ERROR: unique index creation on IVM columns is not supported +DROP MATERIALIZED VIEW mv_ivm_unique; +-- TRUNCATE a base table in join views +BEGIN; +TRUNCATE mv_base_a; +SELECT * FROM mv_ivm_1; + i | j | k +---+---+--- +(0 rows) + +ROLLBACK; +BEGIN; +TRUNCATE mv_base_b; +SELECT * FROM mv_ivm_1; + i | j | k +---+---+--- +(0 rows) + +ROLLBACK; +-- some query syntax +BEGIN; +CREATE FUNCTION ivm_func() RETURNS int LANGUAGE 'sql' + AS 'SELECT 1' IMMUTABLE; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_func AS SELECT * FROM ivm_func(); +NOTICE: could not create an index on materialized view "mv_ivm_func" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_no_tbl AS SELECT 1; +NOTICE: could not create an index on materialized view "mv_ivm_no_tbl" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +ROLLBACK; +-- result of materialized view have DISTINCT clause or the duplicate result. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_duplicate AS SELECT j FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_duplicate" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_distinct AS SELECT DISTINCT j FROM mv_base_a; +NOTICE: created index "mv_ivm_distinct_index" on materialized view "mv_ivm_distinct" +INSERT INTO mv_base_a VALUES(6,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; + j +---- + 10 + 20 + 20 + 30 + 40 + 50 +(6 rows) + +SELECT * FROM mv_ivm_distinct ORDER BY 1; + j +---- + 10 + 20 + 30 + 40 + 50 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; + j +---- + 10 + 20 + 30 + 40 + 50 +(5 rows) + +SELECT * FROM mv_ivm_distinct ORDER BY 1; + j +---- + 10 + 20 + 30 + 40 + 50 +(5 rows) + +ROLLBACK; +-- support SUM(), COUNT() and AVG() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(i), AVG(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 120 | 2 | 60.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +UPDATE mv_base_a SET j = 200 WHERE (i,j) = (2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+---------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 220 | 2 | 110.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) = (2,200); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +ROLLBACK; +-- support COUNT(*) aggregate function +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; + i | sum | count +---+-----+------- + 1 | 10 | 1 + 2 | 20 | 1 + 3 | 30 | 1 + 4 | 40 | 1 + 5 | 50 | 1 +(5 rows) + +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; + i | sum | count +---+-----+------- + 1 | 10 | 1 + 2 | 120 | 2 + 3 | 30 | 1 + 4 | 40 | 1 + 5 | 50 | 1 +(5 rows) + +ROLLBACK; +-- TRUNCATE a base table in aggregate views +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +TRUNCATE mv_base_a; +SELECT sum, count FROM mv_ivm_agg; + sum | count +-----+------- +(0 rows) + +SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; + i | sum | count +---+-----+------- +(0 rows) + +ROLLBACK; +-- support aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_group" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_group ORDER BY 1; + sum | count | avg +-----+-------+--------------------- + 150 | 5 | 30.0000000000000000 +(1 row) + +INSERT INTO mv_base_a VALUES(6,60); +SELECT * FROM mv_ivm_group ORDER BY 1; + sum | count | avg +-----+-------+--------------------- + 210 | 6 | 35.0000000000000000 +(1 row) + +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_group ORDER BY 1; + sum | count | avg +-----+-------+----- + | 0 | +(1 row) + +ROLLBACK; +-- TRUNCATE a base table in aggregate views without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_group" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +TRUNCATE mv_base_a; +SELECT sum, count, avg FROM mv_ivm_group; + sum | count | avg +-----+-------+----- + | 0 | +(1 row) + +SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; + sum | count | avg +-----+-------+----- + | 0 | +(1 row) + +ROLLBACK; +-- resolved issue: When use AVG() function and values is indivisible, result of AVG() is incorrect. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_avg_bug AS SELECT i, SUM(j), COUNT(j), AVG(j) FROM mv_base_A GROUP BY i; +NOTICE: created index "mv_ivm_avg_bug_index" on materialized view "mv_ivm_avg_bug" +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +INSERT INTO mv_base_a VALUES + (1,0), + (1,0), + (2,30), + (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 3 | 3.3333333333333333 + 2 | 80 | 3 | 26.6666666666666667 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) = (1,0); +DELETE FROM mv_base_a WHERE (i,j) = (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; + i | sum | count | avg +---+-----+-------+--------------------- + 1 | 10 | 1 | 10.0000000000000000 + 2 | 20 | 1 | 20.0000000000000000 + 3 | 30 | 1 | 30.0000000000000000 + 4 | 40 | 1 | 40.0000000000000000 + 5 | 50 | 1 | 50.0000000000000000 +(5 rows) + +ROLLBACK; +-- support MIN(), MAX() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT i, MIN(j), MAX(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_min_max_index" on materialized view "mv_ivm_min_max" +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; + i | min | max +---+-----+----- + 1 | 10 | 10 + 2 | 20 | 20 + 3 | 30 | 30 + 4 | 40 | 40 + 5 | 50 | 50 +(5 rows) + +INSERT INTO mv_base_a VALUES + (1,11), (1,12), + (2,21), (2,22), + (3,31), (3,32), + (4,41), (4,42), + (5,51), (5,52); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; + i | min | max +---+-----+----- + 1 | 10 | 12 + 2 | 20 | 22 + 3 | 30 | 32 + 4 | 40 | 42 + 5 | 50 | 52 +(5 rows) + +DELETE FROM mv_base_a WHERE (i,j) IN ((1,10), (2,21), (3,32)); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; + i | min | max +---+-----+----- + 1 | 11 | 12 + 2 | 20 | 22 + 3 | 30 | 31 + 4 | 40 | 42 + 5 | 50 | 52 +(5 rows) + +ROLLBACK; +-- support MIN(), MAX() aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT MIN(j), MAX(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_min_max" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + 10 | 50 +(1 row) + +INSERT INTO mv_base_a VALUES + (0,0), (6,60), (7,70); +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + 0 | 70 +(1 row) + +DELETE FROM mv_base_a WHERE (i,j) IN ((0,0), (7,70)); +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + 10 | 60 +(1 row) + +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_min_max; + min | max +-----+----- + | +(1 row) + +ROLLBACK; +-- Test MIN/MAX after search_path change +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min AS SELECT MIN(j) FROM mv_base_a; +NOTICE: could not create an index on materialized view "mv_ivm_min" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_ivm_min; + min +----- + 10 +(1 row) + +CREATE SCHEMA myschema; +GRANT ALL ON SCHEMA myschema TO public; +CREATE TABLE myschema.mv_base_a (j int); +INSERT INTO myschema.mv_base_a VALUES (1); +DELETE FROM mv_base_a WHERE (i,j) = (1,10); +SELECT * FROM mv_ivm_min; + min +----- + 20 +(1 row) + +SET search_path TO myschema,public,pg_catalog; +DELETE FROM public.mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_min; + min +----- + 30 +(1 row) + +ROLLBACK; +-- aggregate views with column names specified +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; + a | sum +---+------ + 1 | 110 + 2 | 2200 + 3 | 300 + 4 | 40 + 5 | 50 +(5 rows) + +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +NOTICE: created index "mv_ivm_agg_index" on materialized view "mv_ivm_agg" +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; + a | b +---+------ + 1 | 110 + 2 | 2200 + 3 | 300 + 4 | 40 + 5 | 50 +(5 rows) + +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b,c) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +ERROR: too many column names were specified +ROLLBACK; +-- support self join view and multiple change on the same table +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1, 10), (2, 20), (3, 30); +CREATE INCREMENTAL MATERIALIZED VIEW mv_self(v1, v2) AS + SELECT t1.v, t2.v FROM base_t AS t1 JOIN base_t AS t2 ON t1.i = t2.i; +NOTICE: could not create an index on materialized view "mv_self" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv_self ORDER BY v1; + v1 | v2 +----+---- + 10 | 10 + 20 | 20 + 30 | 30 +(3 rows) + +INSERT INTO base_t VALUES (4,40); +DELETE FROM base_t WHERE i = 1; +UPDATE base_t SET v = v*10 WHERE i=2; +SELECT * FROM mv_self ORDER BY v1; + v1 | v2 +-----+----- + 30 | 30 + 40 | 40 + 200 | 200 +(3 rows) + +WITH + ins_t1 AS (INSERT INTO base_t VALUES (5,50) RETURNING 1), + ins_t2 AS (INSERT INTO base_t VALUES (6,60) RETURNING 1), + upd_t AS (UPDATE base_t SET v = v + 100 RETURNING 1), + dlt_t AS (DELETE FROM base_t WHERE i IN (4,5) RETURNING 1) +SELECT NULL; + ?column? +---------- + +(1 row) + +SELECT * FROM mv_self ORDER BY v1; + v1 | v2 +-----+----- + 50 | 50 + 60 | 60 + 130 | 130 + 300 | 300 +(4 rows) + +--- with sub-transactions +SAVEPOINT p1; +INSERT INTO base_t VALUES (7,70); +RELEASE SAVEPOINT p1; +INSERT INTO base_t VALUES (7,77); +SELECT * FROM mv_self ORDER BY v1, v2; + v1 | v2 +-----+----- + 50 | 50 + 60 | 60 + 70 | 70 + 70 | 77 + 77 | 70 + 77 | 77 + 130 | 130 + 300 | 300 +(8 rows) + +ROLLBACK; +-- support simultaneous table changes +BEGIN; +CREATE TABLE base_r (i int, v int); +CREATE TABLE base_s (i int, v int); +INSERT INTO base_r VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO base_s VALUES (1, 100), (2, 200), (3, 300); +CREATE INCREMENTAL MATERIALIZED VIEW mv(v1, v2) AS + SELECT r.v, s.v FROM base_r AS r JOIN base_s AS s USING(i); +NOTICE: could not create an index on materialized view "mv" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv ORDER BY v1; + v1 | v2 +----+----- + 10 | 100 + 20 | 200 + 30 | 300 +(3 rows) + +WITH + ins_r AS (INSERT INTO base_r VALUES (1,11) RETURNING 1), + ins_r2 AS (INSERT INTO base_r VALUES (3,33) RETURNING 1), + ins_s AS (INSERT INTO base_s VALUES (2,222) RETURNING 1), + upd_r AS (UPDATE base_r SET v = v + 1000 WHERE i = 2 RETURNING 1), + dlt_s AS (DELETE FROM base_s WHERE i = 3 RETURNING 1) +SELECT NULL; + ?column? +---------- + +(1 row) + +SELECT * FROM mv ORDER BY v1; + v1 | v2 +------+----- + 10 | 100 + 11 | 100 + 1020 | 200 + 1020 | 222 +(4 rows) + +ROLLBACK; +-- support foreign reference constraints +BEGIN; +CREATE TABLE ri1 (i int PRIMARY KEY); +CREATE TABLE ri2 (i int PRIMARY KEY REFERENCES ri1(i) ON UPDATE CASCADE ON DELETE CASCADE, v int); +INSERT INTO ri1 VALUES (1),(2),(3); +INSERT INTO ri2 VALUES (1),(2),(3); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ri(i1, i2) AS + SELECT ri1.i, ri2.i FROM ri1 JOIN ri2 USING(i); +NOTICE: created index "mv_ri_index" on materialized view "mv_ri" +SELECT * FROM mv_ri ORDER BY i1; + i1 | i2 +----+---- + 1 | 1 + 2 | 2 + 3 | 3 +(3 rows) + +UPDATE ri1 SET i=10 where i=1; +DELETE FROM ri1 WHERE i=2; +SELECT * FROM mv_ri ORDER BY i2; + i1 | i2 +----+---- + 3 | 3 + 10 | 10 +(2 rows) + +ROLLBACK; +-- views including NULL +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1,10),(2, NULL); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +NOTICE: could not create an index on materialized view "mv" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv ORDER BY i; + i | v +---+---- + 1 | 10 + 2 | +(2 rows) + +UPDATE base_t SET v = 20 WHERE i = 2; +SELECT * FROM mv ORDER BY i; + i | v +---+---- + 1 | 10 + 2 | 20 +(2 rows) + +ROLLBACK; +BEGIN; +CREATE TABLE base_t (i int); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +NOTICE: could not create an index on materialized view "mv" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT * FROM mv ORDER BY i; + i +--- +(0 rows) + +INSERT INTO base_t VALUES (1),(NULL); +SELECT * FROM mv ORDER BY i; + i +--- + 1 + +(2 rows) + +ROLLBACK; +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (1, 10), (1, 20); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, sum(v) FROM base_t GROUP BY i; +NOTICE: created index "mv_index" on materialized view "mv" +SELECT * FROM mv ORDER BY i; + i | sum +---+----- + 1 | 30 + | 3 +(2 rows) + +UPDATE base_t SET v = v * 10; +SELECT * FROM mv ORDER BY i; + i | sum +---+----- + 1 | 300 + | 30 +(2 rows) + +ROLLBACK; +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (NULL, 3), (NULL, 4), (NULL, 5); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, min(v), max(v) FROM base_t GROUP BY i; +NOTICE: created index "mv_index" on materialized view "mv" +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 1 | 5 +(1 row) + +DELETE FROM base_t WHERE v = 1; +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 2 | 5 +(1 row) + +DELETE FROM base_t WHERE v = 3; +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 2 | 5 +(1 row) + +DELETE FROM base_t WHERE v = 5; +SELECT * FROM mv ORDER BY i; + i | min | max +---+-----+----- + | 2 | 4 +(1 row) + +ROLLBACK; +-- IMMV containing user defined type +BEGIN; +CREATE TYPE mytype; +CREATE FUNCTION mytype_in(cstring) + RETURNS mytype AS 'int4in' + LANGUAGE INTERNAL STRICT IMMUTABLE; +NOTICE: return type mytype is only a shell +CREATE FUNCTION mytype_out(mytype) + RETURNS cstring AS 'int4out' + LANGUAGE INTERNAL STRICT IMMUTABLE; +NOTICE: argument type mytype is only a shell +CREATE TYPE mytype ( + LIKE = int4, + INPUT = mytype_in, + OUTPUT = mytype_out +); +CREATE FUNCTION mytype_eq(mytype, mytype) + RETURNS bool AS 'int4eq' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_lt(mytype, mytype) + RETURNS bool AS 'int4lt' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_cmp(mytype, mytype) + RETURNS integer AS 'btint4cmp' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE OPERATOR = ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_eq); +CREATE OPERATOR < ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_lt); +CREATE OPERATOR CLASS mytype_ops + DEFAULT FOR TYPE mytype USING btree AS + OPERATOR 1 <, + OPERATOR 3 = , + FUNCTION 1 mytype_cmp(mytype,mytype); +CREATE TABLE t_mytype (x mytype); +CREATE INCREMENTAL MATERIALIZED VIEW mv_mytype AS + SELECT * FROM t_mytype; +NOTICE: could not create an index on materialized view "mv_mytype" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +INSERT INTO t_mytype VALUES ('1'::mytype); +SELECT * FROM mv_mytype; + x +--- + 1 +(1 row) + +ROLLBACK; +-- outer join is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv(a,b) AS SELECT a.i, b.i FROM mv_base_a a LEFT JOIN mv_base_b b ON a.i=b.i; +ERROR: OUTER JOIN is not supported on incrementally maintainable materialized view +-- CTE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv AS + WITH b AS ( SELECT * FROM mv_base_b) SELECT a.i,a.j FROM mv_base_a a, b WHERE a.i = b.i; +ERROR: CTE is not supported on incrementally maintainable materialized view +-- contain system column +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm01 AS SELECT i,j,xmin FROM mv_base_a; +ERROR: system column is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm02 AS SELECT i,j FROM mv_base_a WHERE xmin = '610'; +ERROR: system column is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT i,j,xmin::text AS x_min FROM mv_base_a; +ERROR: system column is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm06 AS SELECT i,j,xidsend(xmin) AS x_min FROM mv_base_a; +ERROR: system column is not supported on incrementally maintainable materialized view +-- contain subquery +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm03 AS SELECT i,j FROM mv_base_a WHERE i IN (SELECT i FROM mv_base_b WHERE k < 103 ); +ERROR: subquery is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT * FROM mv_base_b) b WHERE a.i = b.i; +ERROR: subquery is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm05 AS SELECT i,j, (SELECT k FROM mv_base_b b WHERE a.i = b.i) FROM mv_base_a a; +ERROR: subquery is not supported on incrementally maintainable materialized view +-- contain ORDER BY +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) ORDER BY i,j,k; +ERROR: ORDER BY clause is not supported on incrementally maintainable materialized view +-- contain HAVING +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) GROUP BY i,j,k HAVING SUM(i) > 5; +ERROR: HAVING clause is not supported on incrementally maintainable materialized view +-- contain view or materialized view +CREATE VIEW b_view AS SELECT i,k FROM mv_base_b; +CREATE MATERIALIZED VIEW b_mview AS SELECT i,k FROM mv_base_b; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT a.i,a.j FROM mv_base_a a,b_view b WHERE a.i = b.i; +ERROR: VIEW or MATERIALIZED VIEW is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT a.i,a.j FROM mv_base_a a,b_mview b WHERE a.i = b.i; +ERROR: VIEW or MATERIALIZED VIEW is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm09 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT i, COUNT(*) FROM mv_base_b GROUP BY i) b WHERE a.i = b.i; +ERROR: subquery is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm10 AS SELECT a.i,a.j FROM mv_base_a a WHERE EXISTS(SELECT 1 FROM mv_base_b b WHERE a.i = b.i) OR a.i > 5; +ERROR: subquery is not supported on incrementally maintainable materialized view +-- contain mutable functions +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm12 AS SELECT i,j FROM mv_base_a WHERE i = random()::int; +ERROR: mutable function is not supported on incrementally maintainable materialized view +HINT: functions must be marked IMMUTABLE +-- LIMIT/OFFSET is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm13 AS SELECT i,j FROM mv_base_a LIMIT 10 OFFSET 5; +ERROR: LIMIT/OFFSET clause is not supported on incrementally maintainable materialized view +-- DISTINCT ON is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm14 AS SELECT DISTINCT ON(i) i, j FROM mv_base_a; +ERROR: DISTINCT ON is not supported on incrementally maintainable materialized view +-- TABLESAMPLE clause is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm15 AS SELECT i, j FROM mv_base_a TABLESAMPLE SYSTEM(50); +ERROR: TABLESAMPLE clause is not supported on incrementally maintainable materialized view +-- window functions are not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm16 AS SELECT *, cume_dist() OVER (ORDER BY i) AS rank FROM mv_base_a; +ERROR: window functions are not supported on incrementally maintainable materialized view +-- aggregate function with some options is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm17 AS SELECT COUNT(*) FILTER(WHERE i < 3) FROM mv_base_a; +ERROR: aggregate function with FILTER clause is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm18 AS SELECT COUNT(DISTINCT i) FROM mv_base_a; +ERROR: aggregate function with DISTINCT arguments is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm19 AS SELECT array_agg(j ORDER BY i DESC) FROM mv_base_a; +ERROR: aggregate function with ORDER clause is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm20 AS SELECT i,SUM(j) FROM mv_base_a GROUP BY GROUPING SETS((i),()); +ERROR: GROUPING SETS, ROLLUP, or CUBE clauses is not supported on incrementally maintainable materialized view +-- inheritance parent is not supported +BEGIN; +CREATE TABLE parent (i int, v int); +CREATE TABLE child_a(options text) INHERITS(parent); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm21 AS SELECT * FROM parent; +ERROR: inheritance parent is not supported on incrementally maintainable materialized view +ROLLBACK; +-- UNION statement is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm22 AS SELECT i,j FROM mv_base_a UNION ALL SELECT i,k FROM mv_base_b;; +ERROR: UNION/INTERSECT/EXCEPT statements are not supported on incrementally maintainable materialized view +-- empty target list is not allowed with IVM +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm25 AS SELECT FROM mv_base_a; +ERROR: empty target list is not supported on incrementally maintainable materialized view +-- FOR UPDATE/SHARE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm26 AS SELECT i,j FROM mv_base_a FOR UPDATE; +ERROR: FOR UPDATE/SHARE clause is not supported on incrementally maintainable materialized view +-- tartget list cannot contain ivm column that start with '__ivm' +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm28 AS SELECT i AS "__ivm_count__" FROM mv_base_a; +ERROR: column name __ivm_count__ is not supported on incrementally maintainable materialized view +-- expressions specified in GROUP BY must appear in the target list. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm29 AS SELECT COUNT(i) FROM mv_base_a GROUP BY i; +ERROR: GROUP BY expression not appearing in select list is not supported on incrementally maintainable materialized view +-- experssions containing an aggregate is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm30 AS SELECT sum(i)*0.5 FROM mv_base_a; +ERROR: expression containing an aggregate in it is not supported on incrementally maintainable materialized view +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm31 AS SELECT sum(i)/sum(j) FROM mv_base_a; +ERROR: expression containing an aggregate in it is not supported on incrementally maintainable materialized view +-- VALUES is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_only_values1 AS values(1); +ERROR: VALUES is not supported on incrementally maintainable materialized view +-- views containing base tables with Row Level Security +DROP USER IF EXISTS ivm_admin; +NOTICE: role "ivm_admin" does not exist, skipping +DROP USER IF EXISTS ivm_user; +NOTICE: role "ivm_user" does not exist, skipping +CREATE USER ivm_admin; +CREATE USER ivm_user; +--- create a table with RLS +SET SESSION AUTHORIZATION ivm_admin; +CREATE TABLE rls_tbl(id int, data text, owner name); +INSERT INTO rls_tbl VALUES + (1,'foo','ivm_user'), + (2,'bar','postgres'); +CREATE TABLE num_tbl(id int, num text); +INSERT INTO num_tbl VALUES + (1,'one'), + (2,'two'), + (3,'three'), + (4,'four'), + (5,'five'), + (6,'six'); +--- Users can access only their own rows +CREATE POLICY rls_tbl_policy ON rls_tbl FOR SELECT TO PUBLIC USING(owner = current_user); +ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; +GRANT ALL on rls_tbl TO PUBLIC; +GRANT ALL on num_tbl TO PUBLIC; +--- create a view owned by ivm_user +SET SESSION AUTHORIZATION ivm_user; +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls AS SELECT * FROM rls_tbl; +NOTICE: could not create an index on materialized view "ivm_rls" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + id | data | owner +----+------+---------- + 1 | foo | ivm_user +(1 row) + +RESET SESSION AUTHORIZATION; +--- inserts rows owned by different users +INSERT INTO rls_tbl VALUES + (3,'baz','ivm_user'), + (4,'qux','postgres'); +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + id | data | owner +----+------+---------- + 1 | foo | ivm_user + 3 | baz | ivm_user +(2 rows) + +--- combination of diffent kinds of commands +WITH + i AS (INSERT INTO rls_tbl VALUES(5,'quux','postgres'), (6,'corge','ivm_user')), + u AS (UPDATE rls_tbl SET owner = 'postgres' WHERE id = 1), + u2 AS (UPDATE rls_tbl SET owner = 'ivm_user' WHERE id = 2) +SELECT; +-- +(1 row) + +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + id | data | owner +----+-------+---------- + 2 | bar | ivm_user + 3 | baz | ivm_user + 6 | corge | ivm_user +(3 rows) + +--- +SET SESSION AUTHORIZATION ivm_user; +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls2 AS SELECT * FROM rls_tbl JOIN num_tbl USING(id); +NOTICE: could not create an index on materialized view "ivm_rls2" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +RESET SESSION AUTHORIZATION; +WITH + x AS (UPDATE rls_tbl SET data = data || '_2' where id in (3,4)), + y AS (UPDATE num_tbl SET num = num || '_2' where id in (3,4)) +SELECT; +-- +(1 row) + +SELECT * FROM ivm_rls2 ORDER BY 1,2,3; + id | data | owner | num +----+-------+----------+--------- + 2 | bar | ivm_user | two + 3 | baz_2 | ivm_user | three_2 + 6 | corge | ivm_user | six +(3 rows) + +-- automatic index creation +BEGIN; +CREATE TABLE base_a (i int primary key, j int); +CREATE TABLE base_b (i int primary key, j int); +--- group by: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx1 AS SELECT i, sum(j) FROM base_a GROUP BY i; +NOTICE: created index "mv_idx1_index" on materialized view "mv_idx1" +--- distinct: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx2 AS SELECT DISTINCT j FROM base_a; +NOTICE: created index "mv_idx2_index" on materialized view "mv_idx2" +--- with all pkey columns: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx3(i_a, i_b) AS SELECT a.i, b.i FROM base_a a, base_b b; +NOTICE: created index "mv_idx3_index" on materialized view "mv_idx3" +--- missing some pkey columns: no index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx4 AS SELECT j FROM base_a; +NOTICE: could not create an index on materialized view "mv_idx4" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx5 AS SELECT a.i, b.j FROM base_a a, base_b b; +NOTICE: could not create an index on materialized view "mv_idx5" automatically +DETAIL: This target list does not have all the primary key columns, or this view does not contain GROUP BY or DISTINCT clause. +HINT: Create an index on the materialized view for efficient incremental maintenance. +-- cleanup +DROP TABLE rls_tbl CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to materialized view ivm_rls +drop cascades to materialized view ivm_rls2 +DROP TABLE num_tbl CASCADE; +DROP USER ivm_user; +DROP USER ivm_admin; +DROP TABLE mv_base_b CASCADE; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to materialized view mv_ivm_1 +drop cascades to view b_view +drop cascades to materialized view b_mview +DROP TABLE mv_base_a CASCADE; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index cf46fa3359..013f5db7fb 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -78,7 +78,7 @@ test: brin_bloom brin_multi # psql depends on create_am # amutils depends on geometry, create_index_spgist, hash_index, brin # ---------- -test: create_table_like alter_generic alter_operator misc async dbsize merge misc_functions sysviews tsrf tid tidscan tidrangescan collate.icu.utf8 incremental_sort create_role +test: create_table_like alter_generic alter_operator misc async dbsize merge misc_functions sysviews tsrf tid tidscan tidrangescan collate.icu.utf8 incremental_sort create_role incremental_matview # collate.*.utf8 tests cannot be run in parallel with each other test: rules psql psql_crosstab amutils stats_ext collate.linux.utf8 collate.windows.win1252 diff --git a/src/test/regress/sql/incremental_matview.sql b/src/test/regress/sql/incremental_matview.sql new file mode 100644 index 0000000000..63e26da767 --- /dev/null +++ b/src/test/regress/sql/incremental_matview.sql @@ -0,0 +1,534 @@ +-- create a table to use as a basis for views and materialized views in various combinations +CREATE TABLE mv_base_a (i int, j int); +INSERT INTO mv_base_a VALUES + (1,10), + (2,20), + (3,30), + (4,40), + (5,50); +CREATE TABLE mv_base_b (i int, k int); +INSERT INTO mv_base_b VALUES + (1,101), + (2,102), + (3,103), + (4,104); + +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_1 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) WITH NO DATA; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +REFRESH MATERIALIZED VIEW mv_ivm_1; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + +-- REFRESH WITH NO DATA +BEGIN; +CREATE FUNCTION dummy_ivm_trigger_func() RETURNS TRIGGER AS $$ + BEGIN + RETURN NULL; + END +$$ language plpgsql; + +CREATE CONSTRAINT TRIGGER dummy_ivm_trigger AFTER INSERT +ON mv_base_a FROM mv_ivm_1 FOR EACH ROW +EXECUTE PROCEDURE dummy_ivm_trigger_func(); + +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; + +REFRESH MATERIALIZED VIEW mv_ivm_1 WITH NO DATA; + +SELECT COUNT(*) +FROM pg_depend pd INNER JOIN pg_trigger pt ON pd.objid = pt.oid +WHERE pd.classid = 'pg_trigger'::regclass AND pd.refobjid = 'mv_ivm_1'::regclass; +ROLLBACK; + +-- immediate maintenance +BEGIN; +INSERT INTO mv_base_b VALUES(5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +UPDATE mv_base_a SET j = 0 WHERE i = 1; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +DELETE FROM mv_base_b WHERE (i,k) = (5,105); +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; +ROLLBACK; +SELECT * FROM mv_ivm_1 ORDER BY 1,2,3; + +-- rename of IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_rename AS SELECT DISTINCT * FROM mv_base_a; +ALTER MATERIALIZED VIEW mv_ivm_rename RENAME COLUMN __ivm_count__ TO xxx; +DROP MATERIALIZED VIEW mv_ivm_rename; + +-- unique index on IVM columns +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_unique AS SELECT DISTINCT * FROM mv_base_a; +CREATE UNIQUE INDEX ON mv_ivm_unique(__ivm_count__); +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__)); +CREATE UNIQUE INDEX ON mv_ivm_unique((__ivm_count__ + 1)); +DROP MATERIALIZED VIEW mv_ivm_unique; + +-- TRUNCATE a base table in join views +BEGIN; +TRUNCATE mv_base_a; +SELECT * FROM mv_ivm_1; +ROLLBACK; + +BEGIN; +TRUNCATE mv_base_b; +SELECT * FROM mv_ivm_1; +ROLLBACK; + +-- some query syntax +BEGIN; +CREATE FUNCTION ivm_func() RETURNS int LANGUAGE 'sql' + AS 'SELECT 1' IMMUTABLE; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_func AS SELECT * FROM ivm_func(); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_no_tbl AS SELECT 1; +ROLLBACK; + +-- result of materialized view have DISTINCT clause or the duplicate result. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_duplicate AS SELECT j FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_distinct AS SELECT DISTINCT j FROM mv_base_a; +INSERT INTO mv_base_a VALUES(6,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; +SELECT * FROM mv_ivm_distinct ORDER BY 1; +DELETE FROM mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_duplicate ORDER BY 1; +SELECT * FROM mv_ivm_distinct ORDER BY 1; +ROLLBACK; + +-- support SUM(), COUNT() and AVG() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(i), AVG(j) FROM mv_base_a GROUP BY i; +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +UPDATE mv_base_a SET j = 200 WHERE (i,j) = (2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +DELETE FROM mv_base_a WHERE (i,j) = (2,200); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3,4; +ROLLBACK; + +-- support COUNT(*) aggregate function +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; +INSERT INTO mv_base_a VALUES(2,100); +SELECT * FROM mv_ivm_agg ORDER BY 1,2,3; +ROLLBACK; + +-- TRUNCATE a base table in aggregate views +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg AS SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +TRUNCATE mv_base_a; +SELECT sum, count FROM mv_ivm_agg; +SELECT i, SUM(j), COUNT(*) FROM mv_base_a GROUP BY i; +ROLLBACK; + +-- support aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +SELECT * FROM mv_ivm_group ORDER BY 1; +INSERT INTO mv_base_a VALUES(6,60); +SELECT * FROM mv_ivm_group ORDER BY 1; +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_group ORDER BY 1; +ROLLBACK; + +-- TRUNCATE a base table in aggregate views without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_group AS SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +TRUNCATE mv_base_a; +SELECT sum, count, avg FROM mv_ivm_group; +SELECT SUM(j), COUNT(j), AVG(j) FROM mv_base_a; +ROLLBACK; + +-- resolved issue: When use AVG() function and values is indivisible, result of AVG() is incorrect. +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_avg_bug AS SELECT i, SUM(j), COUNT(j), AVG(j) FROM mv_base_A GROUP BY i; +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; +INSERT INTO mv_base_a VALUES + (1,0), + (1,0), + (2,30), + (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; +DELETE FROM mv_base_a WHERE (i,j) = (1,0); +DELETE FROM mv_base_a WHERE (i,j) = (2,30); +SELECT * FROM mv_ivm_avg_bug ORDER BY 1,2,3; +ROLLBACK; + +-- support MIN(), MAX() aggregate functions +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT i, MIN(j), MAX(j) FROM mv_base_a GROUP BY i; +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; +INSERT INTO mv_base_a VALUES + (1,11), (1,12), + (2,21), (2,22), + (3,31), (3,32), + (4,41), (4,42), + (5,51), (5,52); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; +DELETE FROM mv_base_a WHERE (i,j) IN ((1,10), (2,21), (3,32)); +SELECT * FROM mv_ivm_min_max ORDER BY 1,2,3; +ROLLBACK; + +-- support MIN(), MAX() aggregate functions without GROUP clause +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min_max AS SELECT MIN(j), MAX(j) FROM mv_base_a; +SELECT * FROM mv_ivm_min_max; +INSERT INTO mv_base_a VALUES + (0,0), (6,60), (7,70); +SELECT * FROM mv_ivm_min_max; +DELETE FROM mv_base_a WHERE (i,j) IN ((0,0), (7,70)); +SELECT * FROM mv_ivm_min_max; +DELETE FROM mv_base_a; +SELECT * FROM mv_ivm_min_max; +ROLLBACK; + +-- Test MIN/MAX after search_path change +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_min AS SELECT MIN(j) FROM mv_base_a; +SELECT * FROM mv_ivm_min; + +CREATE SCHEMA myschema; +GRANT ALL ON SCHEMA myschema TO public; +CREATE TABLE myschema.mv_base_a (j int); +INSERT INTO myschema.mv_base_a VALUES (1); + +DELETE FROM mv_base_a WHERE (i,j) = (1,10); +SELECT * FROM mv_ivm_min; + +SET search_path TO myschema,public,pg_catalog; +DELETE FROM public.mv_base_a WHERE (i,j) = (2,20); +SELECT * FROM mv_ivm_min; +ROLLBACK; + +-- aggregate views with column names specified +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +INSERT INTO mv_base_a VALUES (1,100), (2,200), (3,300); +UPDATE mv_base_a SET j = 2000 WHERE (i,j) = (2,20); +DELETE FROM mv_base_a WHERE (i,j) = (3,30); +SELECT * FROM mv_ivm_agg ORDER BY 1,2; +ROLLBACK; +BEGIN; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_agg(a,b,c) AS SELECT i, SUM(j) FROM mv_base_a GROUP BY i; +ROLLBACK; + +-- support self join view and multiple change on the same table +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1, 10), (2, 20), (3, 30); +CREATE INCREMENTAL MATERIALIZED VIEW mv_self(v1, v2) AS + SELECT t1.v, t2.v FROM base_t AS t1 JOIN base_t AS t2 ON t1.i = t2.i; +SELECT * FROM mv_self ORDER BY v1; +INSERT INTO base_t VALUES (4,40); +DELETE FROM base_t WHERE i = 1; +UPDATE base_t SET v = v*10 WHERE i=2; +SELECT * FROM mv_self ORDER BY v1; +WITH + ins_t1 AS (INSERT INTO base_t VALUES (5,50) RETURNING 1), + ins_t2 AS (INSERT INTO base_t VALUES (6,60) RETURNING 1), + upd_t AS (UPDATE base_t SET v = v + 100 RETURNING 1), + dlt_t AS (DELETE FROM base_t WHERE i IN (4,5) RETURNING 1) +SELECT NULL; +SELECT * FROM mv_self ORDER BY v1; + +--- with sub-transactions +SAVEPOINT p1; +INSERT INTO base_t VALUES (7,70); +RELEASE SAVEPOINT p1; +INSERT INTO base_t VALUES (7,77); +SELECT * FROM mv_self ORDER BY v1, v2; + +ROLLBACK; + +-- support simultaneous table changes +BEGIN; +CREATE TABLE base_r (i int, v int); +CREATE TABLE base_s (i int, v int); +INSERT INTO base_r VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO base_s VALUES (1, 100), (2, 200), (3, 300); +CREATE INCREMENTAL MATERIALIZED VIEW mv(v1, v2) AS + SELECT r.v, s.v FROM base_r AS r JOIN base_s AS s USING(i); +SELECT * FROM mv ORDER BY v1; +WITH + ins_r AS (INSERT INTO base_r VALUES (1,11) RETURNING 1), + ins_r2 AS (INSERT INTO base_r VALUES (3,33) RETURNING 1), + ins_s AS (INSERT INTO base_s VALUES (2,222) RETURNING 1), + upd_r AS (UPDATE base_r SET v = v + 1000 WHERE i = 2 RETURNING 1), + dlt_s AS (DELETE FROM base_s WHERE i = 3 RETURNING 1) +SELECT NULL; +SELECT * FROM mv ORDER BY v1; +ROLLBACK; + +-- support foreign reference constraints +BEGIN; +CREATE TABLE ri1 (i int PRIMARY KEY); +CREATE TABLE ri2 (i int PRIMARY KEY REFERENCES ri1(i) ON UPDATE CASCADE ON DELETE CASCADE, v int); +INSERT INTO ri1 VALUES (1),(2),(3); +INSERT INTO ri2 VALUES (1),(2),(3); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ri(i1, i2) AS + SELECT ri1.i, ri2.i FROM ri1 JOIN ri2 USING(i); +SELECT * FROM mv_ri ORDER BY i1; +UPDATE ri1 SET i=10 where i=1; +DELETE FROM ri1 WHERE i=2; +SELECT * FROM mv_ri ORDER BY i2; +ROLLBACK; + +-- views including NULL +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (1,10),(2, NULL); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +SELECT * FROM mv ORDER BY i; +UPDATE base_t SET v = 20 WHERE i = 2; +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +BEGIN; +CREATE TABLE base_t (i int); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT * FROM base_t; +SELECT * FROM mv ORDER BY i; +INSERT INTO base_t VALUES (1),(NULL); +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (1, 10), (1, 20); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, sum(v) FROM base_t GROUP BY i; +SELECT * FROM mv ORDER BY i; +UPDATE base_t SET v = v * 10; +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +BEGIN; +CREATE TABLE base_t (i int, v int); +INSERT INTO base_t VALUES (NULL, 1), (NULL, 2), (NULL, 3), (NULL, 4), (NULL, 5); +CREATE INCREMENTAL MATERIALIZED VIEW mv AS SELECT i, min(v), max(v) FROM base_t GROUP BY i; +SELECT * FROM mv ORDER BY i; +DELETE FROM base_t WHERE v = 1; +SELECT * FROM mv ORDER BY i; +DELETE FROM base_t WHERE v = 3; +SELECT * FROM mv ORDER BY i; +DELETE FROM base_t WHERE v = 5; +SELECT * FROM mv ORDER BY i; +ROLLBACK; + +-- IMMV containing user defined type +BEGIN; + +CREATE TYPE mytype; +CREATE FUNCTION mytype_in(cstring) + RETURNS mytype AS 'int4in' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_out(mytype) + RETURNS cstring AS 'int4out' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE TYPE mytype ( + LIKE = int4, + INPUT = mytype_in, + OUTPUT = mytype_out +); + +CREATE FUNCTION mytype_eq(mytype, mytype) + RETURNS bool AS 'int4eq' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_lt(mytype, mytype) + RETURNS bool AS 'int4lt' + LANGUAGE INTERNAL STRICT IMMUTABLE; +CREATE FUNCTION mytype_cmp(mytype, mytype) + RETURNS integer AS 'btint4cmp' + LANGUAGE INTERNAL STRICT IMMUTABLE; + +CREATE OPERATOR = ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_eq); +CREATE OPERATOR < ( + leftarg = mytype, rightarg = mytype, + procedure = mytype_lt); + +CREATE OPERATOR CLASS mytype_ops + DEFAULT FOR TYPE mytype USING btree AS + OPERATOR 1 <, + OPERATOR 3 = , + FUNCTION 1 mytype_cmp(mytype,mytype); + +CREATE TABLE t_mytype (x mytype); +CREATE INCREMENTAL MATERIALIZED VIEW mv_mytype AS + SELECT * FROM t_mytype; +INSERT INTO t_mytype VALUES ('1'::mytype); +SELECT * FROM mv_mytype; + +ROLLBACK; + +-- outer join is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv(a,b) AS SELECT a.i, b.i FROM mv_base_a a LEFT JOIN mv_base_b b ON a.i=b.i; +-- CTE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv AS + WITH b AS ( SELECT * FROM mv_base_b) SELECT a.i,a.j FROM mv_base_a a, b WHERE a.i = b.i; +-- contain system column +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm01 AS SELECT i,j,xmin FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm02 AS SELECT i,j FROM mv_base_a WHERE xmin = '610'; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT i,j,xmin::text AS x_min FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm06 AS SELECT i,j,xidsend(xmin) AS x_min FROM mv_base_a; +-- contain subquery +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm03 AS SELECT i,j FROM mv_base_a WHERE i IN (SELECT i FROM mv_base_b WHERE k < 103 ); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm04 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT * FROM mv_base_b) b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm05 AS SELECT i,j, (SELECT k FROM mv_base_b b WHERE a.i = b.i) FROM mv_base_a a; +-- contain ORDER BY +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) ORDER BY i,j,k; +-- contain HAVING +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT i,j,k FROM mv_base_a a INNER JOIN mv_base_b b USING(i) GROUP BY i,j,k HAVING SUM(i) > 5; + +-- contain view or materialized view +CREATE VIEW b_view AS SELECT i,k FROM mv_base_b; +CREATE MATERIALIZED VIEW b_mview AS SELECT i,k FROM mv_base_b; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm07 AS SELECT a.i,a.j FROM mv_base_a a,b_view b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm08 AS SELECT a.i,a.j FROM mv_base_a a,b_mview b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm09 AS SELECT a.i,a.j FROM mv_base_a a, (SELECT i, COUNT(*) FROM mv_base_b GROUP BY i) b WHERE a.i = b.i; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm10 AS SELECT a.i,a.j FROM mv_base_a a WHERE EXISTS(SELECT 1 FROM mv_base_b b WHERE a.i = b.i) OR a.i > 5; + +-- contain mutable functions +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm12 AS SELECT i,j FROM mv_base_a WHERE i = random()::int; + +-- LIMIT/OFFSET is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm13 AS SELECT i,j FROM mv_base_a LIMIT 10 OFFSET 5; + +-- DISTINCT ON is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm14 AS SELECT DISTINCT ON(i) i, j FROM mv_base_a; + +-- TABLESAMPLE clause is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm15 AS SELECT i, j FROM mv_base_a TABLESAMPLE SYSTEM(50); + +-- window functions are not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm16 AS SELECT *, cume_dist() OVER (ORDER BY i) AS rank FROM mv_base_a; + +-- aggregate function with some options is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm17 AS SELECT COUNT(*) FILTER(WHERE i < 3) FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm18 AS SELECT COUNT(DISTINCT i) FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm19 AS SELECT array_agg(j ORDER BY i DESC) FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm20 AS SELECT i,SUM(j) FROM mv_base_a GROUP BY GROUPING SETS((i),()); + +-- inheritance parent is not supported +BEGIN; +CREATE TABLE parent (i int, v int); +CREATE TABLE child_a(options text) INHERITS(parent); +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm21 AS SELECT * FROM parent; +ROLLBACK; + +-- UNION statement is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm22 AS SELECT i,j FROM mv_base_a UNION ALL SELECT i,k FROM mv_base_b;; + +-- empty target list is not allowed with IVM +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm25 AS SELECT FROM mv_base_a; + +-- FOR UPDATE/SHARE is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm26 AS SELECT i,j FROM mv_base_a FOR UPDATE; + +-- tartget list cannot contain ivm column that start with '__ivm' +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm28 AS SELECT i AS "__ivm_count__" FROM mv_base_a; + +-- expressions specified in GROUP BY must appear in the target list. +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm29 AS SELECT COUNT(i) FROM mv_base_a GROUP BY i; + +-- experssions containing an aggregate is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm30 AS SELECT sum(i)*0.5 FROM mv_base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm31 AS SELECT sum(i)/sum(j) FROM mv_base_a; + +-- VALUES is not supported +CREATE INCREMENTAL MATERIALIZED VIEW mv_ivm_only_values1 AS values(1); + +-- views containing base tables with Row Level Security +DROP USER IF EXISTS ivm_admin; +DROP USER IF EXISTS ivm_user; +CREATE USER ivm_admin; +CREATE USER ivm_user; + +--- create a table with RLS +SET SESSION AUTHORIZATION ivm_admin; +CREATE TABLE rls_tbl(id int, data text, owner name); +INSERT INTO rls_tbl VALUES + (1,'foo','ivm_user'), + (2,'bar','postgres'); +CREATE TABLE num_tbl(id int, num text); +INSERT INTO num_tbl VALUES + (1,'one'), + (2,'two'), + (3,'three'), + (4,'four'), + (5,'five'), + (6,'six'); + +--- Users can access only their own rows +CREATE POLICY rls_tbl_policy ON rls_tbl FOR SELECT TO PUBLIC USING(owner = current_user); +ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; +GRANT ALL on rls_tbl TO PUBLIC; +GRANT ALL on num_tbl TO PUBLIC; + +--- create a view owned by ivm_user +SET SESSION AUTHORIZATION ivm_user; + +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls AS SELECT * FROM rls_tbl; +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; +RESET SESSION AUTHORIZATION; + +--- inserts rows owned by different users +INSERT INTO rls_tbl VALUES + (3,'baz','ivm_user'), + (4,'qux','postgres'); +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + +--- combination of diffent kinds of commands +WITH + i AS (INSERT INTO rls_tbl VALUES(5,'quux','postgres'), (6,'corge','ivm_user')), + u AS (UPDATE rls_tbl SET owner = 'postgres' WHERE id = 1), + u2 AS (UPDATE rls_tbl SET owner = 'ivm_user' WHERE id = 2) +SELECT; +SELECT id, data, owner FROM ivm_rls ORDER BY 1,2,3; + +--- +SET SESSION AUTHORIZATION ivm_user; +CREATE INCREMENTAL MATERIALIZED VIEW ivm_rls2 AS SELECT * FROM rls_tbl JOIN num_tbl USING(id); +RESET SESSION AUTHORIZATION; + +WITH + x AS (UPDATE rls_tbl SET data = data || '_2' where id in (3,4)), + y AS (UPDATE num_tbl SET num = num || '_2' where id in (3,4)) +SELECT; +SELECT * FROM ivm_rls2 ORDER BY 1,2,3; + +-- automatic index creation +BEGIN; +CREATE TABLE base_a (i int primary key, j int); +CREATE TABLE base_b (i int primary key, j int); + +--- group by: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx1 AS SELECT i, sum(j) FROM base_a GROUP BY i; + +--- distinct: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx2 AS SELECT DISTINCT j FROM base_a; + +--- with all pkey columns: create an index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx3(i_a, i_b) AS SELECT a.i, b.i FROM base_a a, base_b b; + +--- missing some pkey columns: no index +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx4 AS SELECT j FROM base_a; +CREATE INCREMENTAL MATERIALIZED VIEW mv_idx5 AS SELECT a.i, b.j FROM base_a a, base_b b; + +-- cleanup + +DROP TABLE rls_tbl CASCADE; +DROP TABLE num_tbl CASCADE; +DROP USER ivm_user; +DROP USER ivm_admin; + +DROP TABLE mv_base_b CASCADE; +DROP TABLE mv_base_a CASCADE; -- 2.25.1 --Multipart=_Thu__1_Jun_2023_23_59_09_+0900_/G5+8nG46.f1T42K Content-Type: text/x-diff; name="v28-0011-Add-documentations-about-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v28-0011-Add-documentations-about-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v38 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 35e124694ab..540d9e3d5c8 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1341,6 +1341,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Wed__1_Jul_2026_00_04_01_+0900_OVSy2WWK_9aByzDJ Content-Type: text/x-diff; name="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v38-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v33 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index d8767137d9..abce06d046 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1309,6 +1309,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 97406b28c9..0ff5e3922b 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -105,6 +133,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -165,6 +194,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -173,17 +205,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2122,6 +2167,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2203,6 +2250,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2232,6 +2290,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2250,10 +2310,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2445,6 +2513,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2547,19 +2679,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2574,10 +2731,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2585,10 +2743,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2771,6 +2944,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2804,6 +3320,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2812,6 +3335,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Tue__2_Jul_2024_17_03_11_+0900_6OrVBZxOB4o_k6A1 Content-Type: text/x-diff; name="v33-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v33-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v32 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index ecec93ec1c..d3cf3df5a7 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1311,6 +1311,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index b7f6c3831b..42ff0ed98f 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -105,6 +133,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -165,6 +194,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -173,17 +205,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2124,6 +2169,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2205,6 +2252,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2234,6 +2292,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2252,10 +2312,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2447,6 +2515,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2549,19 +2681,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2576,10 +2733,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2587,10 +2745,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2773,6 +2946,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2806,6 +3322,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2814,6 +3337,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Sun__31_Mar_2024_22_59_31_+0900_msknEviJj08_wgqO Content-Type: text/x-diff; name="v32-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v32-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v34 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index d8767137d9..abce06d046 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1309,6 +1309,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index ca6d236989..8cd7ed3b68 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -105,6 +133,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -165,6 +194,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -173,17 +205,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2124,6 +2169,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2205,6 +2252,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2234,6 +2292,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2252,10 +2312,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2447,6 +2515,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2549,19 +2681,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2576,10 +2733,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2587,10 +2745,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2773,6 +2946,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2806,6 +3322,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2814,6 +3337,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.34.1 --Multipart=_Thu__11_Jul_2024_13_23_57_+0900_hKsU2G3a3BgS2FFs Content-Type: text/x-diff; name="v34-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v34-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v30 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index d93eec3eec..0536e44b9e 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1312,6 +1312,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 3c523991ed..e60f92226c 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -73,6 +73,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -109,6 +137,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -169,6 +198,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -177,17 +209,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2123,6 +2168,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2204,6 +2251,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2233,6 +2291,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2251,10 +2311,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2446,6 +2514,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2548,19 +2680,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2575,10 +2732,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2586,10 +2744,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2772,6 +2945,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2805,6 +3321,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2813,6 +3336,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Mon__4_Mar_2024_11_58_46_+0900_UaponF/qQhQrVCFt Content-Type: text/x-diff; name="v30-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v30-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v31 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index aa8440b4e1..7e1627ad1b 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1309,6 +1309,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 97406b28c9..0ff5e3922b 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -105,6 +133,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -165,6 +194,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -173,17 +205,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2122,6 +2167,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2203,6 +2250,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2232,6 +2290,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2250,10 +2310,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2445,6 +2513,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2547,19 +2679,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2574,10 +2731,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2585,10 +2743,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2771,6 +2944,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2804,6 +3320,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2812,6 +3335,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Fri__29_Mar_2024_23_47_00_+0900_KGpmmDOIs1266Ib1 Content-Type: text/x-diff; name="v31-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v31-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v29 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index c8aa558f2e..c40ea6b2bc 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1312,6 +1312,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index ee41f0007d..eff512d40c 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -73,6 +73,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -109,6 +137,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -169,6 +198,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -177,17 +209,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2101,6 +2146,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2182,6 +2229,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2211,6 +2269,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2229,10 +2289,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2424,6 +2492,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2526,19 +2658,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2553,10 +2710,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2564,10 +2722,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2750,6 +2923,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2783,6 +3299,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2791,6 +3314,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Mon__28_Aug_2023_11_52_52_+0900_hj6L5h176QaSGtg7 Content-Type: text/x-diff; name="v29-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v29-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v29 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index c8aa558f2e..c40ea6b2bc 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1312,6 +1312,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index ee41f0007d..eff512d40c 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -73,6 +73,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -109,6 +137,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -169,6 +198,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -177,17 +209,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2101,6 +2146,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2182,6 +2229,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2211,6 +2269,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2229,10 +2289,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2424,6 +2492,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2526,19 +2658,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2553,10 +2710,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2564,10 +2722,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2750,6 +2923,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2783,6 +3299,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2791,6 +3314,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Mon__28_Aug_2023_16_05_30_+0900_b1OvQD_3A3ZMTGvj Content-Type: text/x-diff; name="v29-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v29-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v37 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 45a30309951..ed67f399b7c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1332,6 +1332,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 710224aa994..3b3871c49e2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -69,6 +69,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + SearchPathMatcher *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -121,6 +149,7 @@ typedef struct DroppedImmvInfo } DroppedImmvInfo; static HTAB *mv_trigger_info = NULL; +static HTAB *mv_query_cache = NULL; static HTAB *dropped_immv = NULL; static HTAB *LastIvmUpdateHash; @@ -210,6 +239,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -218,17 +250,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort, SubTransactionId subxid); static void setLastUpdateXid(Oid immv_oid, FullTransactionId xid); @@ -2343,6 +2388,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2424,6 +2471,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2453,6 +2511,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2471,10 +2531,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2666,6 +2734,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2768,19 +2900,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2795,10 +2952,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2806,10 +2964,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2992,6 +3165,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -3025,6 +3541,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -3039,6 +3562,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + SearchPathMatchesCurrentEnvironment(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetSearchPathMatcher(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.43.0 --Multipart=_Fri__29_May_2026_23_14_17_+0900_Te0o73X2VqYK57Gd Content-Type: text/x-diff; name="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Disposition: attachment; filename="v37-0008-Add-aggregates-support-in-IVM.patch" Content-Transfer-Encoding: quoted-printable ^ permalink raw reply [nested|flat] 259+ messages in thread
* [PATCH v28 09/11] Add support for min/max aggregates for IVM @ 2023-05-31 11:58 Yugo Nagata <[email protected]> 0 siblings, 0 replies; 259+ messages in thread From: Yugo Nagata @ 2023-05-31 11:58 UTC (permalink / raw) Supporting min and max is more complicated than count, sum, or avg. For an example of min, when tuples are inserted, the current min value in the view and the min value in the inseteted tuples are compared, then the smaller one is used as the latest min value. On the other hand, when tuples are deleted, if the current min value in the view equals to the min in the deleted tuples, we need re-computation the latest min value from base tables. Otherwise, the current value in the view remains. --- src/backend/commands/createas.c | 45 +++ src/backend/commands/matview.c | 644 +++++++++++++++++++++++++++++++- 2 files changed, 680 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index c8aa558f2e..c40ea6b2bc 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -1312,6 +1312,51 @@ check_aggregate_supports_ivm(Oid aggfnoid) case F_AVG_FLOAT8: case F_AVG_INTERVAL: + /* min */ + case F_MIN_ANYARRAY: + case F_MIN_INT8: + case F_MIN_INT4: + case F_MIN_INT2: + case F_MIN_OID: + case F_MIN_FLOAT4: + case F_MIN_FLOAT8: + case F_MIN_DATE: + case F_MIN_TIME: + case F_MIN_TIMETZ: + case F_MIN_MONEY: + case F_MIN_TIMESTAMP: + case F_MIN_TIMESTAMPTZ: + case F_MIN_INTERVAL: + case F_MIN_TEXT: + case F_MIN_NUMERIC: + case F_MIN_BPCHAR: + case F_MIN_TID: + case F_MIN_ANYENUM: + case F_MIN_INET: + case F_MIN_PG_LSN: + + /* max */ + case F_MAX_ANYARRAY: + case F_MAX_INT8: + case F_MAX_INT4: + case F_MAX_INT2: + case F_MAX_OID: + case F_MAX_FLOAT4: + case F_MAX_FLOAT8: + case F_MAX_DATE: + case F_MAX_TIME: + case F_MAX_TIMETZ: + case F_MAX_MONEY: + case F_MAX_TIMESTAMP: + case F_MAX_TIMESTAMPTZ: + case F_MAX_INTERVAL: + case F_MAX_TEXT: + case F_MAX_NUMERIC: + case F_MAX_BPCHAR: + case F_MAX_TID: + case F_MAX_ANYENUM: + case F_MAX_INET: + case F_MAX_PG_LSN: return true; default: diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index aa6bf2694a..fa65e4aeb0 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -73,6 +73,34 @@ typedef struct #define MV_INIT_QUERYHASHSIZE 16 +/* MV query type codes */ +#define MV_PLAN_RECALC 1 +#define MV_PLAN_SET_VALUE 2 + +/* + * MI_QueryKey + * + * The key identifying a prepared SPI plan in our query hashtable + */ +typedef struct MV_QueryKey +{ + Oid matview_id; /* OID of materialized view */ + int32 query_type; /* query type ID, see MV_PLAN_XXX above */ +} MV_QueryKey; + +/* + * MV_QueryHashEntry + * + * Hash entry for cached plans used to maintain materialized views. + */ +typedef struct MV_QueryHashEntry +{ + MV_QueryKey key; + SPIPlanPtr plan; + OverrideSearchPath *search_path; /* search_path used for parsing + * and planning */ +} MV_QueryHashEntry; + /* * MV_TriggerHashEntry * @@ -109,6 +137,7 @@ typedef struct MV_TriggerTable TupleTableSlot *slot; /* for checking visibility in the pre-state table */ } MV_TriggerTable; +static HTAB *mv_query_cache = NULL; static HTAB *mv_trigger_info = NULL; static bool in_delta_calculation = false; @@ -169,6 +198,9 @@ static void append_set_clause_for_sum(const char *resname, StringInfo buf_old, static void append_set_clause_for_avg(const char *resname, StringInfo buf_old, StringInfo buf_new, StringInfo aggs_list, const char *aggtype); +static void append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min); static char *get_operation_string(IvmOp op, const char *col, const char *arg1, const char *arg2, const char* count_col, const char *castType); static char *get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, @@ -177,17 +209,30 @@ static void apply_old_delta(const char *matviewname, const char *deltaname_old, List *keys); static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname); + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc); static void apply_new_delta(const char *matviewname, const char *deltaname_new, StringInfo target_list); static void apply_new_delta_with_count(const char *matviewname, const char* deltaname_new, List *keys, StringInfo target_list, StringInfo aggs_set, const char* count_colname); static char *get_matching_condition_string(List *keys); +static char *get_returning_string(List *minmax_list, List *is_min_list, List *keys); +static char *get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list); +static char *get_select_for_recalc_string(List *keys); +static void recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel); +static SPIPlanPtr get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes); +static SPIPlanPtr get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes); static void generate_equal(StringInfo querybuf, Oid opttype, const char *leftop, const char *rightop); static void mv_InitHashTables(void); +static SPIPlanPtr mv_FetchPreparedPlan(MV_QueryKey *key); +static void mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan); +static void mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type); static void clean_up_IVM_hash_entry(MV_TriggerHashEntry *entry, bool is_abort); /* @@ -2102,6 +2147,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n ListCell *lc; int i; List *keys = NIL; + List *minmax_list = NIL; + List *is_min_list = NIL; /* @@ -2183,6 +2230,17 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n append_set_clause_for_avg(resname, aggs_set_old, aggs_set_new, aggs_list_buf, format_type_be(aggref->aggtype)); + /* min/max */ + else if (!strcmp(aggname, "min") || !strcmp(aggname, "max")) + { + bool is_min = (!strcmp(aggname, "min")); + + append_set_clause_for_minmax(resname, aggs_set_old, aggs_set_new, aggs_list_buf, is_min); + + /* make a resname list of min and max aggregates */ + minmax_list = lappend(minmax_list, resname); + is_min_list = lappend_int(is_min_list, is_min); + } else elog(ERROR, "unsupported aggregate function: %s", aggname); } @@ -2212,6 +2270,8 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n if (old_tuplestores && tuplestore_tuple_count(old_tuplestores) > 0) { EphemeralNamedRelation enr = palloc(sizeof(EphemeralNamedRelationData)); + SPITupleTable *tuptable_recalc = NULL; + uint64 num_recalc; int rc; /* convert tuplestores to ENR, and register for SPI */ @@ -2230,10 +2290,18 @@ apply_delta(Oid matviewOid, Tuplestorestate *old_tuplestores, Tuplestorestate *n /* apply old delta and get rows to be recalculated */ apply_old_delta_with_count(matviewname, OLD_DELTA_ENRNAME, keys, aggs_list_buf, aggs_set_old, - count_colname); + minmax_list, is_min_list, + count_colname, &tuptable_recalc, &num_recalc); else apply_old_delta(matviewname, OLD_DELTA_ENRNAME, keys); + /* + * If we have min or max, we might have to recalculate aggregate values from base tables + * on some tuples. TIDs and keys such tuples are returned as a result of the above query. + */ + if (minmax_list && tuptable_recalc) + recalc_and_set_values(tuptable_recalc, num_recalc, minmax_list, keys, matviewRel); + } /* For tuple insertion */ if (new_tuplestores && tuplestore_tuple_count(new_tuplestores) > 0) @@ -2425,6 +2493,70 @@ append_set_clause_for_avg(const char *resname, StringInfo buf_old, ); } +/* + * append_set_clause_for_minmax + * + * Append SET clause string for min or max aggregation to given buffers. + * Also, append resnames required for calculating the aggregate value. + * is_min is true if this is min, false if not. + */ +static void +append_set_clause_for_minmax(const char *resname, StringInfo buf_old, + StringInfo buf_new, StringInfo aggs_list, + bool is_min) +{ + char *count_col = IVM_colname("count", resname); + + /* For tuple deletion */ + if (buf_old) + { + /* + * If the new value doesn't became NULL then use the value remaining + * in the view although this will be recomputated afterwords. + */ + appendStringInfo(buf_old, + ", %s = CASE WHEN %s THEN NULL ELSE %s END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_SUB, "mv", "t", count_col), + quote_qualified_identifier("mv", resname) + ); + /* count = mv.count - t.count */ + appendStringInfo(buf_old, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_SUB, count_col, "mv", "t", NULL, NULL) + ); + } + /* For tuple insertion */ + if (buf_new) + { + /* + * min = LEAST(mv.min, diff.min) + * max = GREATEST(mv.max, diff.max) + */ + appendStringInfo(buf_new, + ", %s = CASE WHEN %s THEN NULL ELSE %s(%s,%s) END", + quote_qualified_identifier(NULL, resname), + get_null_condition_string(IVM_ADD, "mv", "diff", count_col), + + is_min ? "LEAST" : "GREATEST", + quote_qualified_identifier("mv", resname), + quote_qualified_identifier("diff", resname) + ); + /* count = mv.count + diff.count */ + appendStringInfo(buf_new, + ", %s = %s", + quote_qualified_identifier(NULL, count_col), + get_operation_string(IVM_ADD, count_col, "mv", "diff", NULL, NULL) + ); + } + + appendStringInfo(aggs_list, ", %s, %s", + quote_qualified_identifier("diff", resname), + quote_qualified_identifier("diff", IVM_colname("count", resname)) + ); +} + /* * get_operation_string * @@ -2527,19 +2659,44 @@ get_null_condition_string(IvmOp op, const char *arg1, const char *arg2, * list to identify a tuple in the view. If the view has aggregates, this * requires strings representing resnames of aggregates and SET clause for * updating aggregate values. + * + * If the view has min or max aggregate, this requires a list of resnames of + * min/max aggregates and a list of boolean which represents which entries in + * minmax_list is min. These are necessary to check if we need to recalculate + * min or max aggregate values. In this case, this query returns TID and keys + * of tuples which need to be recalculated. This result and the number of rows + * are stored in tuptables and num_recalc repectedly. + * */ static void apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, List *keys, StringInfo aggs_list, StringInfo aggs_set, - const char *count_colname) + List *minmax_list, List *is_min_list, + const char *count_colname, + SPITupleTable **tuptable_recalc, uint64 *num_recalc) { StringInfoData querybuf; char *match_cond; + char *updt_returning = ""; + char *select_for_recalc = "SELECT"; bool agg_without_groupby = (list_length(keys) == 0); + Assert(tuptable_recalc != NULL); + Assert(num_recalc != NULL); + /* build WHERE condition for searching tuples to be deleted */ match_cond = get_matching_condition_string(keys); + /* + * We need a special RETURNING clause and SELECT statement for min/max to + * check which tuple needs re-calculation from base tables. + */ + if (minmax_list) + { + updt_returning = get_returning_string(minmax_list, is_min_list, keys); + select_for_recalc = get_select_for_recalc_string(keys); + } + /* Search for matching tuples from the view and update or delete if found. */ initStringInfo(&querybuf); appendStringInfo(&querybuf, @@ -2554,10 +2711,11 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, "UPDATE %s AS mv SET %s = mv.%s OPERATOR(pg_catalog.-) t.%s " "%s" /* SET clauses for aggregates */ "FROM t WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND NOT for_dlt " - ")" - /* delete a tuple if this is to be deleted */ - "DELETE FROM %s AS mv USING t " - "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt", + "%s" /* RETURNING clause for recalc infomation */ + "), dlt AS (" /* delete a tuple if this is to be deleted */ + "DELETE FROM %s AS mv USING t " + "WHERE mv.ctid OPERATOR(pg_catalog.=) t.ctid AND for_dlt" + ") %s", /* SELECT returning which tuples need to be recalculated */ count_colname, count_colname, count_colname, (agg_without_groupby ? "false" : "true"), (aggs_list != NULL ? aggs_list->data : ""), @@ -2565,10 +2723,25 @@ apply_old_delta_with_count(const char *matviewname, const char *deltaname_old, match_cond, matviewname, count_colname, count_colname, count_colname, (aggs_set != NULL ? aggs_set->data : ""), - matviewname); + updt_returning, + matviewname, + select_for_recalc); - if (SPI_exec(querybuf.data, 0) != SPI_OK_DELETE) + if (SPI_exec(querybuf.data, 0) != SPI_OK_SELECT) elog(ERROR, "SPI_exec failed: %s", querybuf.data); + + + /* Return tuples to be recalculated. */ + if (minmax_list) + { + *tuptable_recalc = SPI_tuptable; + *num_recalc = SPI_processed; + } + else + { + *tuptable_recalc = NULL; + *num_recalc = 0; + } } /* @@ -2751,6 +2924,349 @@ get_matching_condition_string(List *keys) return match_cond.data; } +/* + * get_returning_string + * + * Build a string for RETURNING clause of UPDATE used in apply_old_delta_with_count. + * This clause returns ctid and a boolean value that indicates if we need to + * recalculate min or max value, for each updated row. + */ +static char * +get_returning_string(List *minmax_list, List *is_min_list, List *keys) +{ + StringInfoData returning; + char *recalc_cond; + ListCell *lc; + + Assert(minmax_list != NIL && is_min_list != NIL); + recalc_cond = get_minmax_recalc_condition_string(minmax_list, is_min_list); + + initStringInfo(&returning); + + appendStringInfo(&returning, "RETURNING mv.ctid AS tid, (%s) AS recalc", recalc_cond); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + appendStringInfo(&returning, ", %s", quote_qualified_identifier("mv", resname)); + } + + return returning.data; +} + +/* + * get_minmax_recalc_condition_string + * + * Build a predicate string for checking if any min/max aggregate + * value needs to be recalculated. + */ +static char * +get_minmax_recalc_condition_string(List *minmax_list, List *is_min_list) +{ + StringInfoData recalc_cond; + ListCell *lc1, *lc2; + + initStringInfo(&recalc_cond); + + Assert (list_length(minmax_list) == list_length(is_min_list)); + + forboth (lc1, minmax_list, lc2, is_min_list) + { + char *resname = (char *) lfirst(lc1); + bool is_min = (bool) lfirst_int(lc2); + char *op_str = (is_min ? ">=" : "<="); + + appendStringInfo(&recalc_cond, "%s OPERATOR(pg_catalog.%s) %s", + quote_qualified_identifier("mv", resname), + op_str, + quote_qualified_identifier("t", resname) + ); + + if (lnext(minmax_list, lc1)) + appendStringInfo(&recalc_cond, " OR "); + } + + return recalc_cond.data; +} + +/* + * get_select_for_recalc_string + * + * Build a query to return tid and keys of tuples which need + * recalculation. This is used as the result of the query + * built by apply_old_delta. + */ +static char * +get_select_for_recalc_string(List *keys) +{ + StringInfoData qry; + ListCell *lc; + + initStringInfo(&qry); + + appendStringInfo(&qry, "SELECT tid"); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + appendStringInfo(&qry, ", %s", NameStr(attr->attname)); + } + + appendStringInfo(&qry, " FROM updt WHERE recalc"); + + return qry.data; +} + +/* + * recalc_and_set_values + * + * Recalculate tuples in a materialized from base tables and update these. + * The tuples which needs recalculation are specified by keys, and resnames + * of columns to be updated are specified by namelist. TIDs and key values + * are given by tuples in tuptable_recalc. Its first attribute must be TID + * and key values must be following this. + */ +static void +recalc_and_set_values(SPITupleTable *tuptable_recalc, int64 num_tuples, + List *namelist, List *keys, Relation matviewRel) +{ + TupleDesc tupdesc_recalc = tuptable_recalc->tupdesc; + Oid *keyTypes = NULL, *types = NULL; + char *keyNulls = NULL, *nulls = NULL; + Datum *keyVals = NULL, *vals = NULL; + int num_vals = list_length(namelist); + int num_keys = list_length(keys); + uint64 i; + Oid matviewOid; + char *matviewname; + + matviewOid = RelationGetRelid(matviewRel); + matviewname = quote_qualified_identifier(get_namespace_name(RelationGetNamespace(matviewRel)), + RelationGetRelationName(matviewRel)); + + /* If we have keys, initialize arrays for them. */ + if (keys) + { + keyTypes = palloc(sizeof(Oid) * num_keys); + keyNulls = palloc(sizeof(char) * num_keys); + keyVals = palloc(sizeof(Datum) * num_keys); + /* a tuple contains keys to be recalculated and ctid to be updated*/ + Assert(tupdesc_recalc->natts == num_keys + 1); + + /* Types of key attributes */ + for (i = 0; i < num_keys; i++) + keyTypes[i] = TupleDescAttr(tupdesc_recalc, i + 1)->atttypid; + } + + /* allocate memory for all attribute names and tid */ + types = palloc(sizeof(Oid) * (num_vals + 1)); + nulls = palloc(sizeof(char) * (num_vals + 1)); + vals = palloc(sizeof(Datum) * (num_vals + 1)); + + /* For each tuple which needs recalculation */ + for (i = 0; i < num_tuples; i++) + { + int j; + bool isnull; + SPIPlanPtr plan; + SPITupleTable *tuptable_newvals; + TupleDesc tupdesc_newvals; + + /* Set group key values as parameters if needed. */ + if (keys) + { + for (j = 0; j < num_keys; j++) + { + keyVals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, j + 2, &isnull); + if (isnull) + keyNulls[j] = 'n'; + else + keyNulls[j] = ' '; + } + } + + /* + * Get recalculated values from base tables. The result must be + * only one tuple thich contains the new values for specified keys. + */ + plan = get_plan_for_recalc(matviewOid, namelist, keys, keyTypes); + if (SPI_execute_plan(plan, keyVals, keyNulls, false, 0) != SPI_OK_SELECT) + elog(ERROR, "SPI_execute_plan"); + if (SPI_processed != 1) + elog(ERROR, "SPI_execute_plan returned zero or more than one rows"); + + tuptable_newvals = SPI_tuptable; + tupdesc_newvals = tuptable_newvals->tupdesc; + + Assert(tupdesc_newvals->natts == num_vals); + + /* Set the new values as parameters */ + for (j = 0; j < tupdesc_newvals->natts; j++) + { + if (i == 0) + types[j] = TupleDescAttr(tupdesc_newvals, j)->atttypid; + + vals[j] = SPI_getbinval(tuptable_newvals->vals[0], tupdesc_newvals, j + 1, &isnull); + if (isnull) + nulls[j] = 'n'; + else + nulls[j] = ' '; + } + /* Set TID of the view tuple to be updated as a parameter */ + types[j] = TIDOID; + vals[j] = SPI_getbinval(tuptable_recalc->vals[i], tupdesc_recalc, 1, &isnull); + nulls[j] = ' '; + + /* Update the view tuple to the new values */ + plan = get_plan_for_set_values(matviewOid, matviewname, namelist, types); + if (SPI_execute_plan(plan, vals, nulls, false, 0) != SPI_OK_UPDATE) + elog(ERROR, "SPI_execute_plan"); + } +} + + +/* + * get_plan_for_recalc + * + * Create or fetch a plan for recalculating value in the view's target list + * from base tables using the definition query of materialized view specified + * by matviewOid. namelist is a list of resnames of values to be recalculated. + * + * keys is a list of keys to identify tuples to be recalculated if this is not + * empty. KeyTypes is an array of types of keys. + */ +static SPIPlanPtr +get_plan_for_recalc(Oid matviewOid, List *namelist, List *keys, Oid *keyTypes) +{ + MV_QueryKey hash_key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the recalculation */ + mv_BuildQueryKey(&hash_key, matviewOid, MV_PLAN_RECALC); + if ((plan = mv_FetchPreparedPlan(&hash_key)) == NULL) + { + ListCell *lc; + StringInfoData str; + char *viewdef; + + /* get view definition of matview */ + viewdef = text_to_cstring((text *) DatumGetPointer( + DirectFunctionCall1(pg_get_viewdef, ObjectIdGetDatum(matviewOid)))); + /* get rid of trailing semi-colon */ + viewdef[strlen(viewdef)-1] = '\0'; + + /* + * Build a query string for recalculating values. This is like + * + * SELECT x1, x2, x3, ... FROM ( ... view definition query ...) mv + * WHERE (key1, key2, ...) = ($1, $2, ...); + */ + + initStringInfo(&str); + appendStringInfo(&str, "SELECT "); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, " FROM (%s) mv", viewdef); + + if (keys) + { + int i = 1; + char paramname[16]; + + appendStringInfo(&str, " WHERE ("); + foreach (lc, keys) + { + Form_pg_attribute attr = (Form_pg_attribute) lfirst(lc); + char *resname = NameStr(attr->attname); + Oid typid = attr->atttypid; + + sprintf(paramname, "$%d", i); + appendStringInfo(&str, "("); + generate_equal(&str, typid, resname, paramname); + appendStringInfo(&str, " OR (%s IS NULL AND %s IS NULL))", + resname, paramname); + + if (lnext(keys, lc)) + appendStringInfoString(&str, " AND "); + i++; + } + appendStringInfo(&str, ")"); + } + else + keyTypes = NULL; + + plan = SPI_prepare(str.data, list_length(keys), keyTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&hash_key, plan); + } + + return plan; +} + +/* + * get_plan_for_set_values + * + * Create or fetch a plan for applying new values calculated by + * get_plan_for_recalc to a materialized view specified by matviewOid. + * matviewname is the name of the view. namelist is a list of resnames + * of attributes to be updated, and valTypes is an array of types of the + * values. + */ +static SPIPlanPtr +get_plan_for_set_values(Oid matviewOid, char *matviewname, List *namelist, + Oid *valTypes) +{ + MV_QueryKey key; + SPIPlanPtr plan; + + /* Fetch or prepare a saved plan for the real check */ + mv_BuildQueryKey(&key, matviewOid, MV_PLAN_SET_VALUE); + if ((plan = mv_FetchPreparedPlan(&key)) == NULL) + { + ListCell *lc; + StringInfoData str; + int i; + + /* + * Build a query string for applying min/max values. This is like + * + * UPDATE matviewname AS mv + * SET (x1, x2, x3, x4) = ($1, $2, $3, $4) + * WHERE ctid = $5; + */ + + initStringInfo(&str); + appendStringInfo(&str, "UPDATE %s AS mv SET (", matviewname); + foreach (lc, namelist) + { + appendStringInfo(&str, "%s", (char *) lfirst(lc)); + if (lnext(namelist, lc)) + appendStringInfoString(&str, ", "); + } + appendStringInfo(&str, ") = ROW("); + + for (i = 1; i <= list_length(namelist); i++) + appendStringInfo(&str, "%s$%d", (i==1 ? "" : ", "), i); + + appendStringInfo(&str, ") WHERE ctid OPERATOR(pg_catalog.=) $%d", i); + + plan = SPI_prepare(str.data, list_length(namelist) + 1, valTypes); + if (plan == NULL) + elog(ERROR, "SPI_prepare returned %s for %s", SPI_result_code_string(SPI_result), str.data); + + SPI_keepplan(plan); + mv_HashPreparedPlan(&key, plan); + } + + return plan; +} + /* * generate_equals * @@ -2784,6 +3300,13 @@ mv_InitHashTables(void) { HASHCTL ctl; + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(MV_QueryKey); + ctl.entrysize = sizeof(MV_QueryHashEntry); + mv_query_cache = hash_create("MV query cache", + MV_INIT_QUERYHASHSIZE, + &ctl, HASH_ELEM | HASH_BLOBS); + memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(MV_TriggerHashEntry); @@ -2792,6 +3315,109 @@ mv_InitHashTables(void) &ctl, HASH_ELEM | HASH_BLOBS); } +/* + * mv_FetchPreparedPlan + */ +static SPIPlanPtr +mv_FetchPreparedPlan(MV_QueryKey *key) +{ + MV_QueryHashEntry *entry; + SPIPlanPtr plan; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Lookup for the key + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_FIND, NULL); + if (entry == NULL) + return NULL; + + /* + * Check whether the plan is still valid. If it isn't, we don't want to + * simply rely on plancache.c to regenerate it; rather we should start + * from scratch and rebuild the query text too. This is to cover cases + * such as table/column renames. We depend on the plancache machinery to + * detect possible invalidations, though. + * + * CAUTION: this check is only trustworthy if the caller has already + * locked both materialized views and base tables. + * + * Also, check whether the search_path is still the same as when we made it. + * If it isn't, we need to rebuild the query text because the result of + * pg_ivm_get_viewdef() will change. + */ + plan = entry->plan; + if (plan && SPI_plan_is_valid(plan) && + OverrideSearchPathMatchesCurrent(entry->search_path)) + return plan; + + /* + * Otherwise we might as well flush the cached plan now, to free a little + * memory space before we make a new one. + */ + if (plan) + SPI_freeplan(plan); + if (entry->search_path) + pfree(entry->search_path); + + entry->plan = NULL; + entry->search_path = NULL; + + return NULL; +} + +/* + * mv_HashPreparedPlan + * + * Add another plan to our private SPI query plan hashtable. + */ +static void +mv_HashPreparedPlan(MV_QueryKey *key, SPIPlanPtr plan) +{ + MV_QueryHashEntry *entry; + bool found; + + /* + * On the first call initialize the hashtable + */ + if (!mv_query_cache) + mv_InitHashTables(); + + /* + * Add the new plan. We might be overwriting an entry previously found + * invalid by mv_FetchPreparedPlan. + */ + entry = (MV_QueryHashEntry *) hash_search(mv_query_cache, + (void *) key, + HASH_ENTER, &found); + Assert(!found || entry->plan == NULL); + entry->plan = plan; + entry->search_path = GetOverrideSearchPath(TopMemoryContext); +} + +/* + * mv_BuildQueryKey + * + * Construct a hashtable key for a prepared SPI plan for IVM. + */ +static void +mv_BuildQueryKey(MV_QueryKey *key, Oid matview_id, int32 query_type) +{ + /* + * We assume struct MV_QueryKey contains no padding bytes, else we'd need + * to use memset to clear them. + */ + key->matview_id = matview_id; + key->query_type = query_type; +} + /* * AtAbort_IVM * -- 2.25.1 --Multipart=_Thu__1_Jun_2023_23_59_09_+0900_/G5+8nG46.f1T42K Content-Type: text/x-diff; name="v28-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Disposition: attachment; filename="v28-0010-Add-regression-tests-for-Incremental-View-Mainte.patch" Content-Transfer-Encoding: 7bit ^ permalink raw reply [nested|flat] 259+ messages in thread
end of thread, other threads:[~2023-05-31 11:58 UTC | newest] Thread overview: 259+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2021-03-10 02:11 [PATCH v28 10/11] Add regression tests for Incremental View Maintenance Takuma Hoshiai <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v32 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v31 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v30 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v29 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v33 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v34 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v28 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v29 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v38 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]> 2023-05-31 11:58 [PATCH v37 09/11] Add support for min/max aggregates for IVM Yugo Nagata <[email protected]>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox