agora inbox for pgsql-hackers@postgresql.orghelp / color / mirror / Atom feed
[PATCH 2/3] Support clauses of the form Var op Var 6+ messages / 2 participants [nested] [flat]
* [PATCH 2/3] Support clauses of the form Var op Var @ 2019-11-11 00:34 Tomas Vondra <tv@fuzzy.cz> 0 siblings, 0 replies; 6+ messages in thread From: Tomas Vondra @ 2019-11-11 00:34 UTC (permalink / raw) --- src/backend/statistics/extended_stats.c | 63 ++++++++++++---- src/backend/statistics/mcv.c | 75 ++++++++++++++++++- .../statistics/extended_stats_internal.h | 2 +- src/test/regress/expected/stats_ext.out | 72 ++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 22 ++++++ 5 files changed, 217 insertions(+), 17 deletions(-) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 24ece6f99c..1872cd4529 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -986,14 +986,18 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, { RangeTblEntry *rte = root->simple_rte_array[relid]; OpExpr *expr = (OpExpr *) clause; - Var *var; + Var *var, + *var2; /* Only expressions with two arguments are considered compatible. */ if (list_length(expr->args) != 2) return false; - /* Check if the expression the right shape (one Var, one Const) */ - if (!examine_opclause_expression(expr, &var, NULL, NULL)) + /* + * Check if the expression the right shape (one Var and one Const, + * or two Vars). + */ + if (!examine_opclause_expression(expr, &var, &var2, NULL, NULL)) return false; /* @@ -1033,7 +1037,20 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, !get_func_leakproof(get_opcode(expr->opno))) return false; - return statext_is_compatible_clause_internal(root, (Node *) var, + /* + * Check compatibility of the first Var - we get this one for both + * types of supported expressions (Var op Const) and (Var op Var). + */ + if (!statext_is_compatible_clause_internal(root, (Node *) var, + relid, attnums)) + return false; + + /* For (Var op Const) we don't get the second Var, and we're done. */ + if (!var2) + return true; + + /* For (Var op Var) check compatibility of the second Var. */ + return statext_is_compatible_clause_internal(root, (Node *) var2, relid, attnums); } @@ -1419,19 +1436,21 @@ statext_clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, * examine_opclause_expression * Split expression into Var and Const parts. * - * Attempts to match the arguments to either (Var op Const) or (Const op Var), - * possibly with a RelabelType on top. When the expression matches this form, - * returns true, otherwise returns false. + * Attempts to match the arguments to either (Var op Const) or (Const op Var) + * or (Var op Var), possibly with a RelabelType on top. When the expression + * matches this form, returns true, otherwise returns false. * * Optionally returns pointers to the extracted Var/Const nodes, when passed * non-null pointers (varp, cstp and varonleftp). The varonleftp flag specifies * on which side of the operator we found the Var node. */ bool -examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonleftp) +examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, + Const **cstp, bool *varonleftp) { - Var *var; - Const *cst; + Var *var1 = NULL; + Var *var2 = NULL; + Const *cst = NULL; bool varonleft; Node *leftop, *rightop; @@ -1451,22 +1470,38 @@ examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonl if (IsA(leftop, Var) && IsA(rightop, Const)) { - var = (Var *) leftop; + var1 = (Var *) leftop; cst = (Const *) rightop; varonleft = true; } else if (IsA(leftop, Const) && IsA(rightop, Var)) { - var = (Var *) rightop; + var1 = (Var *) rightop; cst = (Const *) leftop; varonleft = false; } + else if (IsA(leftop, Var) && IsA(rightop, Var)) + { + var1 = (Var *) leftop; + var2 = (Var *) rightop; + varonleft = false; + + /* + * Both variables have to be for the same relation (otherwise it's + * a join clause, and we don't deal with those yet. + */ + if (var1->varno != var2->varno) + return false; + } else return false; /* return pointers to the extracted parts if requested */ - if (varp) - *varp = var; + if (var1p) + *var1p = var1; + + if (var2p) + *var2p = var2; if (cstp) *cstp = cst; diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 3f42713aa2..97d3083451 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1581,16 +1581,25 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, /* valid only after examine_opclause_expression returns true */ Var *var; + Var *var2; Const *cst; bool varonleft; fmgr_info(get_opcode(expr->opno), &opproc); - /* extract the var and const from the expression */ - if (examine_opclause_expression(expr, &var, &cst, &varonleft)) + /* extract the vars and const from the expression */ + if (!examine_opclause_expression(expr, &var, &var2, &cst, &varonleft)) + continue; /* XXX Can this actually happen? */ + + /* We should always get at least one Var. */ + Assert(var); + + if (cst) { int idx; + Assert(!var2); + /* match the attribute to a dimension of the statistic */ idx = bms_member_index(keys, var->varattno); @@ -1651,6 +1660,68 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, matches[i] = RESULT_MERGE(matches[i], is_or, match); } } + else + { + int idx; + int idx2; + + Assert(var2); + + /* match the attribute to a dimension of the statistic */ + idx = bms_member_index(keys, var->varattno); + idx2 = bms_member_index(keys, var2->varattno); + + /* + * Walk through the MCV items and evaluate the current clause. + * We can skip items that were already ruled out, and + * terminate if there are no remaining MCV items that might + * possibly match. + */ + for (i = 0; i < mcvlist->nitems; i++) + { + bool match = true; + MCVItem *item = &mcvlist->items[i]; + + /* + * When either of the MCV items is NULL we can treat this + * as a mismatch. We must not call the operator because + * of strictness. + */ + if (item->isnull[idx] || item->isnull[idx2]) + { + matches[i] = RESULT_MERGE(matches[i], is_or, false); + continue; + } + + /* + * Skip MCV items that can't change result in the bitmap. + * Once the value gets false for AND-lists, or true for + * OR-lists, we don't need to look at more clauses. + */ + if (RESULT_IS_FINAL(matches[i], is_or)) + continue; + + /* + * First check whether the constant is below the lower + * boundary (in that case we can skip the bucket, because + * there's no overlap). + * + * We don't store collations used to build the statistics, + * but we can use the collation for the attribute itself, + * as stored in varcollid. We do reset the statistics after + * a type change (including collation change), so this is + * OK. We may need to relax this after allowing extended + * statistics on expressions. + */ + match = DatumGetBool(FunctionCall2Coll(&opproc, + var->varcollid, + item->values[idx], + item->values[idx2])); + + /* update the match bitmap with the result */ + matches[i] = RESULT_MERGE(matches[i], is_or, match); + } + } } else if (IsA(clause, NullTest)) { diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 5171895bba..804089bc57 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -96,7 +96,7 @@ extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows, TupleDesc tdesc, MultiSortSupport mss, int numattrs, AttrNumber *attnums); -extern bool examine_opclause_expression(OpExpr *expr, Var **varp, +extern bool examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, Const **cstp, bool *varonleftp); extern Selectivity mcv_clauselist_selectivity(PlannerInfo *root, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 5344b70cf4..4c078ae61f 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -603,6 +603,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 343 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 1667 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1667 | 0 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; ANALYZE mcv_lists; @@ -654,6 +666,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 200 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 3750 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1 | 0 +(1 row) + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); SELECT d.stxdmcv IS NOT NULL @@ -749,6 +773,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 3750 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 25 | 2500 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; ANALYZE mcv_lists; @@ -758,6 +788,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 2500 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -808,6 +844,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1094 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 9950 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 50 | 2500 +(1 row) + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; ANALYZE mcv_lists_bool; @@ -835,6 +883,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -869,6 +929,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AN 4 | 142 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 1 | 5000 +(1 row) + -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; CREATE STATISTICS mcv_lists_multi_2 (mcv) ON c, d FROM mcv_lists_multi; @@ -891,6 +957,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AN 143 | 142 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 5000 | 5000 +(1 row) + DROP TABLE mcv_lists_multi; -- Permission tests. Users should not be able to see specific data values in -- the extended statistics, if they lack permission to see those values in diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index fa989fccb0..b7519b275b 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -381,6 +381,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; @@ -402,6 +406,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); @@ -473,6 +481,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; @@ -480,6 +490,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -521,6 +533,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; @@ -534,6 +550,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -556,6 +576,7 @@ ANALYZE mcv_lists_multi; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; @@ -566,6 +587,7 @@ ANALYZE mcv_lists_multi; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); DROP TABLE mcv_lists_multi; -- 2.21.1 --n6bnrzck2js7zcqs-- ^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH 2/3] Support clauses of the form Var op Var @ 2019-11-11 00:34 Tomas Vondra <tv@fuzzy.cz> 0 siblings, 0 replies; 6+ messages in thread From: Tomas Vondra @ 2019-11-11 00:34 UTC (permalink / raw) --- src/backend/statistics/extended_stats.c | 63 ++++++++++++---- src/backend/statistics/mcv.c | 75 ++++++++++++++++++- .../statistics/extended_stats_internal.h | 2 +- src/test/regress/expected/stats_ext.out | 72 ++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 22 ++++++ 5 files changed, 217 insertions(+), 17 deletions(-) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 24ece6f99c..1872cd4529 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -986,14 +986,18 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, { RangeTblEntry *rte = root->simple_rte_array[relid]; OpExpr *expr = (OpExpr *) clause; - Var *var; + Var *var, + *var2; /* Only expressions with two arguments are considered compatible. */ if (list_length(expr->args) != 2) return false; - /* Check if the expression the right shape (one Var, one Const) */ - if (!examine_opclause_expression(expr, &var, NULL, NULL)) + /* + * Check if the expression the right shape (one Var and one Const, + * or two Vars). + */ + if (!examine_opclause_expression(expr, &var, &var2, NULL, NULL)) return false; /* @@ -1033,7 +1037,20 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, !get_func_leakproof(get_opcode(expr->opno))) return false; - return statext_is_compatible_clause_internal(root, (Node *) var, + /* + * Check compatibility of the first Var - we get this one for both + * types of supported expressions (Var op Const) and (Var op Var). + */ + if (!statext_is_compatible_clause_internal(root, (Node *) var, + relid, attnums)) + return false; + + /* For (Var op Const) we don't get the second Var, and we're done. */ + if (!var2) + return true; + + /* For (Var op Var) check compatibility of the second Var. */ + return statext_is_compatible_clause_internal(root, (Node *) var2, relid, attnums); } @@ -1419,19 +1436,21 @@ statext_clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, * examine_opclause_expression * Split expression into Var and Const parts. * - * Attempts to match the arguments to either (Var op Const) or (Const op Var), - * possibly with a RelabelType on top. When the expression matches this form, - * returns true, otherwise returns false. + * Attempts to match the arguments to either (Var op Const) or (Const op Var) + * or (Var op Var), possibly with a RelabelType on top. When the expression + * matches this form, returns true, otherwise returns false. * * Optionally returns pointers to the extracted Var/Const nodes, when passed * non-null pointers (varp, cstp and varonleftp). The varonleftp flag specifies * on which side of the operator we found the Var node. */ bool -examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonleftp) +examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, + Const **cstp, bool *varonleftp) { - Var *var; - Const *cst; + Var *var1 = NULL; + Var *var2 = NULL; + Const *cst = NULL; bool varonleft; Node *leftop, *rightop; @@ -1451,22 +1470,38 @@ examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonl if (IsA(leftop, Var) && IsA(rightop, Const)) { - var = (Var *) leftop; + var1 = (Var *) leftop; cst = (Const *) rightop; varonleft = true; } else if (IsA(leftop, Const) && IsA(rightop, Var)) { - var = (Var *) rightop; + var1 = (Var *) rightop; cst = (Const *) leftop; varonleft = false; } + else if (IsA(leftop, Var) && IsA(rightop, Var)) + { + var1 = (Var *) leftop; + var2 = (Var *) rightop; + varonleft = false; + + /* + * Both variables have to be for the same relation (otherwise it's + * a join clause, and we don't deal with those yet. + */ + if (var1->varno != var2->varno) + return false; + } else return false; /* return pointers to the extracted parts if requested */ - if (varp) - *varp = var; + if (var1p) + *var1p = var1; + + if (var2p) + *var2p = var2; if (cstp) *cstp = cst; diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 3f42713aa2..97d3083451 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1581,16 +1581,25 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, /* valid only after examine_opclause_expression returns true */ Var *var; + Var *var2; Const *cst; bool varonleft; fmgr_info(get_opcode(expr->opno), &opproc); - /* extract the var and const from the expression */ - if (examine_opclause_expression(expr, &var, &cst, &varonleft)) + /* extract the vars and const from the expression */ + if (!examine_opclause_expression(expr, &var, &var2, &cst, &varonleft)) + continue; /* XXX Can this actually happen? */ + + /* We should always get at least one Var. */ + Assert(var); + + if (cst) { int idx; + Assert(!var2); + /* match the attribute to a dimension of the statistic */ idx = bms_member_index(keys, var->varattno); @@ -1651,6 +1660,68 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, matches[i] = RESULT_MERGE(matches[i], is_or, match); } } + else + { + int idx; + int idx2; + + Assert(var2); + + /* match the attribute to a dimension of the statistic */ + idx = bms_member_index(keys, var->varattno); + idx2 = bms_member_index(keys, var2->varattno); + + /* + * Walk through the MCV items and evaluate the current clause. + * We can skip items that were already ruled out, and + * terminate if there are no remaining MCV items that might + * possibly match. + */ + for (i = 0; i < mcvlist->nitems; i++) + { + bool match = true; + MCVItem *item = &mcvlist->items[i]; + + /* + * When either of the MCV items is NULL we can treat this + * as a mismatch. We must not call the operator because + * of strictness. + */ + if (item->isnull[idx] || item->isnull[idx2]) + { + matches[i] = RESULT_MERGE(matches[i], is_or, false); + continue; + } + + /* + * Skip MCV items that can't change result in the bitmap. + * Once the value gets false for AND-lists, or true for + * OR-lists, we don't need to look at more clauses. + */ + if (RESULT_IS_FINAL(matches[i], is_or)) + continue; + + /* + * First check whether the constant is below the lower + * boundary (in that case we can skip the bucket, because + * there's no overlap). + * + * We don't store collations used to build the statistics, + * but we can use the collation for the attribute itself, + * as stored in varcollid. We do reset the statistics after + * a type change (including collation change), so this is + * OK. We may need to relax this after allowing extended + * statistics on expressions. + */ + match = DatumGetBool(FunctionCall2Coll(&opproc, + var->varcollid, + item->values[idx], + item->values[idx2])); + + /* update the match bitmap with the result */ + matches[i] = RESULT_MERGE(matches[i], is_or, match); + } + } } else if (IsA(clause, NullTest)) { diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 5171895bba..804089bc57 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -96,7 +96,7 @@ extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows, TupleDesc tdesc, MultiSortSupport mss, int numattrs, AttrNumber *attnums); -extern bool examine_opclause_expression(OpExpr *expr, Var **varp, +extern bool examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, Const **cstp, bool *varonleftp); extern Selectivity mcv_clauselist_selectivity(PlannerInfo *root, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 5344b70cf4..4c078ae61f 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -603,6 +603,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 343 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 1667 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1667 | 0 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; ANALYZE mcv_lists; @@ -654,6 +666,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 200 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 3750 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1 | 0 +(1 row) + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); SELECT d.stxdmcv IS NOT NULL @@ -749,6 +773,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 3750 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 25 | 2500 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; ANALYZE mcv_lists; @@ -758,6 +788,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 2500 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -808,6 +844,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1094 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 9950 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 50 | 2500 +(1 row) + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; ANALYZE mcv_lists_bool; @@ -835,6 +883,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -869,6 +929,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AN 4 | 142 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 1 | 5000 +(1 row) + -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; CREATE STATISTICS mcv_lists_multi_2 (mcv) ON c, d FROM mcv_lists_multi; @@ -891,6 +957,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AN 143 | 142 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 5000 | 5000 +(1 row) + DROP TABLE mcv_lists_multi; -- Permission tests. Users should not be able to see specific data values in -- the extended statistics, if they lack permission to see those values in diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index fa989fccb0..b7519b275b 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -381,6 +381,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; @@ -402,6 +406,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); @@ -473,6 +481,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; @@ -480,6 +490,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -521,6 +533,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; @@ -534,6 +550,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -556,6 +576,7 @@ ANALYZE mcv_lists_multi; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; @@ -566,6 +587,7 @@ ANALYZE mcv_lists_multi; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); DROP TABLE mcv_lists_multi; -- 2.21.1 --mxrkhfy6o2lpe2eo Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="0003-Support-for-extended-statistics-on-expressi-20200306.patch" ^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH 2/3] Support clauses of the form Var op Var @ 2019-11-11 00:34 Tomas Vondra <tv@fuzzy.cz> 0 siblings, 0 replies; 6+ messages in thread From: Tomas Vondra @ 2019-11-11 00:34 UTC (permalink / raw) --- src/backend/statistics/extended_stats.c | 63 ++++++++++++---- src/backend/statistics/mcv.c | 75 ++++++++++++++++++- .../statistics/extended_stats_internal.h | 2 +- src/test/regress/expected/stats_ext.out | 72 ++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 22 ++++++ 5 files changed, 217 insertions(+), 17 deletions(-) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 24ece6f99c..1872cd4529 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -986,14 +986,18 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, { RangeTblEntry *rte = root->simple_rte_array[relid]; OpExpr *expr = (OpExpr *) clause; - Var *var; + Var *var, + *var2; /* Only expressions with two arguments are considered compatible. */ if (list_length(expr->args) != 2) return false; - /* Check if the expression the right shape (one Var, one Const) */ - if (!examine_opclause_expression(expr, &var, NULL, NULL)) + /* + * Check if the expression the right shape (one Var and one Const, + * or two Vars). + */ + if (!examine_opclause_expression(expr, &var, &var2, NULL, NULL)) return false; /* @@ -1033,7 +1037,20 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, !get_func_leakproof(get_opcode(expr->opno))) return false; - return statext_is_compatible_clause_internal(root, (Node *) var, + /* + * Check compatibility of the first Var - we get this one for both + * types of supported expressions (Var op Const) and (Var op Var). + */ + if (!statext_is_compatible_clause_internal(root, (Node *) var, + relid, attnums)) + return false; + + /* For (Var op Const) we don't get the second Var, and we're done. */ + if (!var2) + return true; + + /* For (Var op Var) check compatibility of the second Var. */ + return statext_is_compatible_clause_internal(root, (Node *) var2, relid, attnums); } @@ -1419,19 +1436,21 @@ statext_clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, * examine_opclause_expression * Split expression into Var and Const parts. * - * Attempts to match the arguments to either (Var op Const) or (Const op Var), - * possibly with a RelabelType on top. When the expression matches this form, - * returns true, otherwise returns false. + * Attempts to match the arguments to either (Var op Const) or (Const op Var) + * or (Var op Var), possibly with a RelabelType on top. When the expression + * matches this form, returns true, otherwise returns false. * * Optionally returns pointers to the extracted Var/Const nodes, when passed * non-null pointers (varp, cstp and varonleftp). The varonleftp flag specifies * on which side of the operator we found the Var node. */ bool -examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonleftp) +examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, + Const **cstp, bool *varonleftp) { - Var *var; - Const *cst; + Var *var1 = NULL; + Var *var2 = NULL; + Const *cst = NULL; bool varonleft; Node *leftop, *rightop; @@ -1451,22 +1470,38 @@ examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonl if (IsA(leftop, Var) && IsA(rightop, Const)) { - var = (Var *) leftop; + var1 = (Var *) leftop; cst = (Const *) rightop; varonleft = true; } else if (IsA(leftop, Const) && IsA(rightop, Var)) { - var = (Var *) rightop; + var1 = (Var *) rightop; cst = (Const *) leftop; varonleft = false; } + else if (IsA(leftop, Var) && IsA(rightop, Var)) + { + var1 = (Var *) leftop; + var2 = (Var *) rightop; + varonleft = false; + + /* + * Both variables have to be for the same relation (otherwise it's + * a join clause, and we don't deal with those yet. + */ + if (var1->varno != var2->varno) + return false; + } else return false; /* return pointers to the extracted parts if requested */ - if (varp) - *varp = var; + if (var1p) + *var1p = var1; + + if (var2p) + *var2p = var2; if (cstp) *cstp = cst; diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 3f42713aa2..97d3083451 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1581,16 +1581,25 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, /* valid only after examine_opclause_expression returns true */ Var *var; + Var *var2; Const *cst; bool varonleft; fmgr_info(get_opcode(expr->opno), &opproc); - /* extract the var and const from the expression */ - if (examine_opclause_expression(expr, &var, &cst, &varonleft)) + /* extract the vars and const from the expression */ + if (!examine_opclause_expression(expr, &var, &var2, &cst, &varonleft)) + continue; /* XXX Can this actually happen? */ + + /* We should always get at least one Var. */ + Assert(var); + + if (cst) { int idx; + Assert(!var2); + /* match the attribute to a dimension of the statistic */ idx = bms_member_index(keys, var->varattno); @@ -1651,6 +1660,68 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, matches[i] = RESULT_MERGE(matches[i], is_or, match); } } + else + { + int idx; + int idx2; + + Assert(var2); + + /* match the attribute to a dimension of the statistic */ + idx = bms_member_index(keys, var->varattno); + idx2 = bms_member_index(keys, var2->varattno); + + /* + * Walk through the MCV items and evaluate the current clause. + * We can skip items that were already ruled out, and + * terminate if there are no remaining MCV items that might + * possibly match. + */ + for (i = 0; i < mcvlist->nitems; i++) + { + bool match = true; + MCVItem *item = &mcvlist->items[i]; + + /* + * When either of the MCV items is NULL we can treat this + * as a mismatch. We must not call the operator because + * of strictness. + */ + if (item->isnull[idx] || item->isnull[idx2]) + { + matches[i] = RESULT_MERGE(matches[i], is_or, false); + continue; + } + + /* + * Skip MCV items that can't change result in the bitmap. + * Once the value gets false for AND-lists, or true for + * OR-lists, we don't need to look at more clauses. + */ + if (RESULT_IS_FINAL(matches[i], is_or)) + continue; + + /* + * First check whether the constant is below the lower + * boundary (in that case we can skip the bucket, because + * there's no overlap). + * + * We don't store collations used to build the statistics, + * but we can use the collation for the attribute itself, + * as stored in varcollid. We do reset the statistics after + * a type change (including collation change), so this is + * OK. We may need to relax this after allowing extended + * statistics on expressions. + */ + match = DatumGetBool(FunctionCall2Coll(&opproc, + var->varcollid, + item->values[idx], + item->values[idx2])); + + /* update the match bitmap with the result */ + matches[i] = RESULT_MERGE(matches[i], is_or, match); + } + } } else if (IsA(clause, NullTest)) { diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 5171895bba..804089bc57 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -96,7 +96,7 @@ extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows, TupleDesc tdesc, MultiSortSupport mss, int numattrs, AttrNumber *attnums); -extern bool examine_opclause_expression(OpExpr *expr, Var **varp, +extern bool examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, Const **cstp, bool *varonleftp); extern Selectivity mcv_clauselist_selectivity(PlannerInfo *root, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 5344b70cf4..4c078ae61f 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -603,6 +603,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 343 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 1667 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1667 | 0 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; ANALYZE mcv_lists; @@ -654,6 +666,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 200 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 3750 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1 | 0 +(1 row) + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); SELECT d.stxdmcv IS NOT NULL @@ -749,6 +773,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 3750 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 25 | 2500 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; ANALYZE mcv_lists; @@ -758,6 +788,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 2500 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -808,6 +844,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1094 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 9950 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 50 | 2500 +(1 row) + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; ANALYZE mcv_lists_bool; @@ -835,6 +883,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -869,6 +929,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AN 4 | 142 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 1 | 5000 +(1 row) + -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; CREATE STATISTICS mcv_lists_multi_2 (mcv) ON c, d FROM mcv_lists_multi; @@ -891,6 +957,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AN 143 | 142 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 5000 | 5000 +(1 row) + DROP TABLE mcv_lists_multi; -- Permission tests. Users should not be able to see specific data values in -- the extended statistics, if they lack permission to see those values in diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index fa989fccb0..b7519b275b 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -381,6 +381,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; @@ -402,6 +406,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); @@ -473,6 +481,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; @@ -480,6 +490,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -521,6 +533,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; @@ -534,6 +550,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -556,6 +576,7 @@ ANALYZE mcv_lists_multi; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; @@ -566,6 +587,7 @@ ANALYZE mcv_lists_multi; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); DROP TABLE mcv_lists_multi; -- 2.21.1 --n6bnrzck2js7zcqs-- ^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH 4/4] Support clauses of the form Var op Var @ 2020-03-08 22:27 Tomas Vondra <tv@fuzzy.cz> 0 siblings, 0 replies; 6+ messages in thread From: Tomas Vondra @ 2020-03-08 22:27 UTC (permalink / raw) --- src/backend/statistics/extended_stats.c | 63 ++++++++++++---- src/backend/statistics/mcv.c | 75 ++++++++++++++++++- .../statistics/extended_stats_internal.h | 2 +- src/test/regress/expected/stats_ext.out | 72 ++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 22 ++++++ 5 files changed, 217 insertions(+), 17 deletions(-) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index daf95ff437..ddcb5d2955 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -986,14 +986,18 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, { RangeTblEntry *rte = root->simple_rte_array[relid]; OpExpr *expr = (OpExpr *) clause; - Var *var; + Var *var, + *var2; /* Only expressions with two arguments are considered compatible. */ if (list_length(expr->args) != 2) return false; - /* Check if the expression the right shape (one Var, one Const) */ - if (!examine_opclause_expression(expr, &var, NULL, NULL)) + /* + * Check if the expression the right shape (one Var and one Const, + * or two Vars). + */ + if (!examine_opclause_expression(expr, &var, &var2, NULL, NULL)) return false; /* @@ -1033,7 +1037,20 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, !get_func_leakproof(get_opcode(expr->opno))) return false; - return statext_is_compatible_clause_internal(root, (Node *) var, + /* + * Check compatibility of the first Var - we get this one for both + * types of supported expressions (Var op Const) and (Var op Var). + */ + if (!statext_is_compatible_clause_internal(root, (Node *) var, + relid, attnums)) + return false; + + /* For (Var op Const) we don't get the second Var, and we're done. */ + if (!var2) + return true; + + /* For (Var op Var) check compatibility of the second Var. */ + return statext_is_compatible_clause_internal(root, (Node *) var2, relid, attnums); } @@ -1422,19 +1439,21 @@ statext_clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, * examine_opclause_expression * Split expression into Var and Const parts. * - * Attempts to match the arguments to either (Var op Const) or (Const op Var), - * possibly with a RelabelType on top. When the expression matches this form, - * returns true, otherwise returns false. + * Attempts to match the arguments to either (Var op Const) or (Const op Var) + * or (Var op Var), possibly with a RelabelType on top. When the expression + * matches this form, returns true, otherwise returns false. * * Optionally returns pointers to the extracted Var/Const nodes, when passed * non-null pointers (varp, cstp and varonleftp). The varonleftp flag specifies * on which side of the operator we found the Var node. */ bool -examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonleftp) +examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, + Const **cstp, bool *varonleftp) { - Var *var; - Const *cst; + Var *var1 = NULL; + Var *var2 = NULL; + Const *cst = NULL; bool varonleft; Node *leftop, *rightop; @@ -1454,22 +1473,38 @@ examine_opclause_expression(OpExpr *expr, Var **varp, Const **cstp, bool *varonl if (IsA(leftop, Var) && IsA(rightop, Const)) { - var = (Var *) leftop; + var1 = (Var *) leftop; cst = (Const *) rightop; varonleft = true; } else if (IsA(leftop, Const) && IsA(rightop, Var)) { - var = (Var *) rightop; + var1 = (Var *) rightop; cst = (Const *) leftop; varonleft = false; } + else if (IsA(leftop, Var) && IsA(rightop, Var)) + { + var1 = (Var *) leftop; + var2 = (Var *) rightop; + varonleft = false; + + /* + * Both variables have to be for the same relation (otherwise it's + * a join clause, and we don't deal with those yet. + */ + if (var1->varno != var2->varno) + return false; + } else return false; /* return pointers to the extracted parts if requested */ - if (varp) - *varp = var; + if (var1p) + *var1p = var1; + + if (var2p) + *var2p = var2; if (cstp) *cstp = cst; diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 3f42713aa2..97d3083451 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1581,16 +1581,25 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, /* valid only after examine_opclause_expression returns true */ Var *var; + Var *var2; Const *cst; bool varonleft; fmgr_info(get_opcode(expr->opno), &opproc); - /* extract the var and const from the expression */ - if (examine_opclause_expression(expr, &var, &cst, &varonleft)) + /* extract the vars and const from the expression */ + if (!examine_opclause_expression(expr, &var, &var2, &cst, &varonleft)) + continue; /* XXX Can this actually happen? */ + + /* We should always get at least one Var. */ + Assert(var); + + if (cst) { int idx; + Assert(!var2); + /* match the attribute to a dimension of the statistic */ idx = bms_member_index(keys, var->varattno); @@ -1651,6 +1660,68 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, matches[i] = RESULT_MERGE(matches[i], is_or, match); } } + else + { + int idx; + int idx2; + + Assert(var2); + + /* match the attribute to a dimension of the statistic */ + idx = bms_member_index(keys, var->varattno); + idx2 = bms_member_index(keys, var2->varattno); + + /* + * Walk through the MCV items and evaluate the current clause. + * We can skip items that were already ruled out, and + * terminate if there are no remaining MCV items that might + * possibly match. + */ + for (i = 0; i < mcvlist->nitems; i++) + { + bool match = true; + MCVItem *item = &mcvlist->items[i]; + + /* + * When either of the MCV items is NULL we can treat this + * as a mismatch. We must not call the operator because + * of strictness. + */ + if (item->isnull[idx] || item->isnull[idx2]) + { + matches[i] = RESULT_MERGE(matches[i], is_or, false); + continue; + } + + /* + * Skip MCV items that can't change result in the bitmap. + * Once the value gets false for AND-lists, or true for + * OR-lists, we don't need to look at more clauses. + */ + if (RESULT_IS_FINAL(matches[i], is_or)) + continue; + + /* + * First check whether the constant is below the lower + * boundary (in that case we can skip the bucket, because + * there's no overlap). + * + * We don't store collations used to build the statistics, + * but we can use the collation for the attribute itself, + * as stored in varcollid. We do reset the statistics after + * a type change (including collation change), so this is + * OK. We may need to relax this after allowing extended + * statistics on expressions. + */ + match = DatumGetBool(FunctionCall2Coll(&opproc, + var->varcollid, + item->values[idx], + item->values[idx2])); + + /* update the match bitmap with the result */ + matches[i] = RESULT_MERGE(matches[i], is_or, match); + } + } } else if (IsA(clause, NullTest)) { diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 5171895bba..804089bc57 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -96,7 +96,7 @@ extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows, TupleDesc tdesc, MultiSortSupport mss, int numattrs, AttrNumber *attnums); -extern bool examine_opclause_expression(OpExpr *expr, Var **varp, +extern bool examine_opclause_expression(OpExpr *expr, Var **var1p, Var **var2p, Const **cstp, bool *varonleftp); extern Selectivity mcv_clauselist_selectivity(PlannerInfo *root, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 383465e8cb..3eb1804cf3 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -603,6 +603,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 343 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 1667 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1667 | 0 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; ANALYZE mcv_lists; @@ -654,6 +666,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 200 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 3750 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1 | 0 +(1 row) + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); SELECT d.stxdmcv IS NOT NULL @@ -749,6 +773,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 3750 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 25 | 2500 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; ANALYZE mcv_lists; @@ -758,6 +788,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d 2500 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -808,6 +844,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1094 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 9950 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 50 | 2500 +(1 row) + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; ANALYZE mcv_lists_bool; @@ -835,6 +883,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -893,6 +953,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR 2649 | 1572 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 1 | 5000 +(1 row) + -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; CREATE STATISTICS mcv_lists_multi_2 (mcv) ON c, d FROM mcv_lists_multi; @@ -939,6 +1005,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR 1571 | 1572 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 5000 | 5000 +(1 row) + DROP TABLE mcv_lists_multi; -- Permission tests. Users should not be able to see specific data values in -- the extended statistics, if they lack permission to see those values in diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 6fd8b016bd..42b67f7cde 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -381,6 +381,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, c FROM mcv_lists; @@ -402,6 +406,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + -- check change of unrelated column type does not reset the MCV statistics ALTER TABLE mcv_lists ALTER COLUMN d TYPE VARCHAR(64); @@ -473,6 +481,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON b, d FROM mcv_lists; @@ -480,6 +490,8 @@ ANALYZE mcv_lists; SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = ''x'' OR d = ''x'''); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- mcv with arrays CREATE TABLE mcv_lists_arrays ( a TEXT[], @@ -521,6 +533,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; @@ -534,6 +550,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -560,6 +580,7 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE b = 0 OR SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE (a = 0 AND b = 0) OR (c = 0 AND d = 0)'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR b = 0 OR c = 0 OR d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; @@ -574,6 +595,7 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE b = 0 OR SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE (a = 0 AND b = 0) OR (c = 0 AND d = 0)'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR b = 0 OR c = 0 OR d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); DROP TABLE mcv_lists_multi; -- 2.21.1 --vcvacfll7i2xqkl4-- ^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH 2/2] Support clauses of the form Var op Var @ 2020-03-17 01:57 Tomas Vondra <tv@fuzzy.cz> 0 siblings, 0 replies; 6+ messages in thread From: Tomas Vondra @ 2020-03-17 01:57 UTC (permalink / raw) --- src/backend/statistics/extended_stats.c | 67 ++++++++++++----- src/backend/statistics/mcv.c | 71 +++++++++++++++++- .../statistics/extended_stats_internal.h | 2 +- src/test/regress/expected/stats_ext.out | 72 +++++++++++++++++++ src/test/regress/sql/stats_ext.sql | 22 ++++++ 5 files changed, 215 insertions(+), 19 deletions(-) diff --git a/src/backend/statistics/extended_stats.c b/src/backend/statistics/extended_stats.c index 93d67791c5..f8947d9097 100644 --- a/src/backend/statistics/extended_stats.c +++ b/src/backend/statistics/extended_stats.c @@ -987,14 +987,18 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, { RangeTblEntry *rte = root->simple_rte_array[relid]; OpExpr *expr = (OpExpr *) clause; - Var *var; + Var *var, + *var2; /* Only expressions with two arguments are considered compatible. */ if (list_length(expr->args) != 2) return false; - /* Check if the expression has the right shape (one Var, one Const) */ - if (!examine_clause_args(expr->args, &var, NULL, NULL)) + /* + * Check if the expression has the right shape (one Var and one Const, + * or two Vars). + */ + if (!examine_clause_args(expr->args, &var, &var2, NULL, NULL)) return false; /* @@ -1034,7 +1038,20 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, !get_func_leakproof(get_opcode(expr->opno))) return false; - return statext_is_compatible_clause_internal(root, (Node *) var, + /* + * Check compatibility of the first Var - we get this one for both + * types of supported expressions (Var op Const) and (Var op Var). + */ + if (!statext_is_compatible_clause_internal(root, (Node *) var, + relid, attnums)) + return false; + + /* For (Var op Const) we don't get the second Var, and we're done. */ + if (!var2) + return true; + + /* For (Var op Var) check compatibility of the second Var. */ + return statext_is_compatible_clause_internal(root, (Node *) var2, relid, attnums); } @@ -1050,7 +1067,7 @@ statext_is_compatible_clause_internal(PlannerInfo *root, Node *clause, return false; /* Check if the expression has the right shape (one Var, one Const) */ - if (!examine_clause_args(expr->args, &var, NULL, NULL)) + if (!examine_clause_args(expr->args, &var, NULL, NULL, NULL)) return false; /* @@ -1456,22 +1473,24 @@ statext_clauselist_selectivity(PlannerInfo *root, List *clauses, int varRelid, } /* - * examine_opclause_expression + * examine_clause_args * Split expression into Var and Const parts. * - * Attempts to match the arguments to either (Var op Const) or (Const op Var), - * possibly with a RelabelType on top. When the expression matches this form, - * returns true, otherwise returns false. + * Attempts to match the arguments to either (Var op Const) or (Const op Var) + * or (Var op Var), possibly with a RelabelType on top. When the expression + * matches this form, returns true, otherwise returns false. * * Optionally returns pointers to the extracted Var/Const nodes, when passed * non-null pointers (varp, cstp and varonleftp). The varonleftp flag specifies * on which side of the operator we found the Var node. */ bool -examine_clause_args(List *args, Var **varp, Const **cstp, bool *varonleftp) +examine_clause_args(List *args, Var **var1p, Var **var2p, Const **cstp, + bool *varonleftp) { - Var *var; - Const *cst; + Var *var1 = NULL; + Var *var2 = NULL; + Const *cst = NULL; bool varonleft; Node *leftop, *rightop; @@ -1491,22 +1510,38 @@ examine_clause_args(List *args, Var **varp, Const **cstp, bool *varonleftp) if (IsA(leftop, Var) && IsA(rightop, Const)) { - var = (Var *) leftop; + var1 = (Var *) leftop; cst = (Const *) rightop; varonleft = true; } else if (IsA(leftop, Const) && IsA(rightop, Var)) { - var = (Var *) rightop; + var1 = (Var *) rightop; cst = (Const *) leftop; varonleft = false; } + else if (IsA(leftop, Var) && IsA(rightop, Var)) + { + var1 = (Var *) leftop; + var2 = (Var *) rightop; + varonleft = false; + + /* + * Both variables have to be for the same relation (otherwise it's + * a join clause, and we don't deal with those yet. + */ + if (var1->varno != var2->varno) + return false; + } else return false; /* return pointers to the extracted parts if requested */ - if (varp) - *varp = var; + if (var1p) + *var1p = var1; + + if (var2p) + *var2p = var2; if (cstp) *cstp = cst; diff --git a/src/backend/statistics/mcv.c b/src/backend/statistics/mcv.c index 343011178b..3e58d36d68 100644 --- a/src/backend/statistics/mcv.c +++ b/src/backend/statistics/mcv.c @@ -1581,13 +1581,17 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, /* valid only after examine_clause_args returns true */ Var *var; + Var *var2; Const *cst; bool varonleft; fmgr_info(get_opcode(expr->opno), &opproc); /* extract the var and const from the expression */ - if (examine_clause_args(expr->args, &var, &cst, &varonleft)) + if (!examine_clause_args(expr->args, &var, &var2, &cst, &varonleft)) + continue; + + if (cst) /* Var op Const */ { int idx; @@ -1651,6 +1655,68 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, matches[i] = RESULT_MERGE(matches[i], is_or, match); } } + else /* Var op Var */ + { + int idx; + int idx2; + + Assert(var2); + + /* match the attribute to a dimension of the statistic */ + idx = bms_member_index(keys, var->varattno); + idx2 = bms_member_index(keys, var2->varattno); + + /* + * Walk through the MCV items and evaluate the current clause. + * We can skip items that were already ruled out, and + * terminate if there are no remaining MCV items that might + * possibly match. + */ + for (i = 0; i < mcvlist->nitems; i++) + { + bool match = true; + MCVItem *item = &mcvlist->items[i]; + + /* + * When either of the MCV items is NULL we can treat this + * as a mismatch. We must not call the operator because + * of strictness. + */ + if (item->isnull[idx] || item->isnull[idx2]) + { + matches[i] = RESULT_MERGE(matches[i], is_or, false); + continue; + } + + /* + * Skip MCV items that can't change result in the bitmap. + * Once the value gets false for AND-lists, or true for + * OR-lists, we don't need to look at more clauses. + */ + if (RESULT_IS_FINAL(matches[i], is_or)) + continue; + + /* + * First check whether the constant is below the lower + * boundary (in that case we can skip the bucket, because + * there's no overlap). + * + * We don't store collations used to build the statistics, + * but we can use the collation for the attribute itself, + * as stored in varcollid. We do reset the statistics after + * a type change (including collation change), so this is + * OK. We may need to relax this after allowing extended + * statistics on expressions. + */ + match = DatumGetBool(FunctionCall2Coll(&opproc, + var->varcollid, + item->values[idx], + item->values[idx2])); + + /* update the match bitmap with the result */ + matches[i] = RESULT_MERGE(matches[i], is_or, match); + } + } } else if (IsA(clause, ScalarArrayOpExpr)) { @@ -1665,7 +1731,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, fmgr_info(get_opcode(expr->opno), &opproc); /* extract the var and const from the expression */ - if (examine_clause_args(expr->args, &var, &cst, &varonleft)) + if (examine_clause_args(expr->args, &var, NULL, &cst, &varonleft)) { int idx; @@ -1679,6 +1745,7 @@ mcv_get_match_bitmap(PlannerInfo *root, List *clauses, /* ScalarArrayOpExpr has the Var always on the left */ Assert(varonleft); + Assert(cst); if (!cst->constisnull) { diff --git a/src/include/statistics/extended_stats_internal.h b/src/include/statistics/extended_stats_internal.h index 6c039a81c2..ee34a897c8 100644 --- a/src/include/statistics/extended_stats_internal.h +++ b/src/include/statistics/extended_stats_internal.h @@ -96,7 +96,7 @@ extern SortItem *build_sorted_items(int numrows, int *nitems, HeapTuple *rows, TupleDesc tdesc, MultiSortSupport mss, int numattrs, AttrNumber *attnums); -extern bool examine_clause_args(List *args, Var **varp, +extern bool examine_clause_args(List *args, Var **var1p, Var **var2p, Const **cstp, bool *varonleftp); extern Selectivity mcv_clauselist_selectivity(PlannerInfo *root, diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out index 3aba4a37c6..481be40141 100644 --- a/src/test/regress/expected/stats_ext.out +++ b/src/test/regress/expected/stats_ext.out @@ -940,6 +940,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 343 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 1667 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1667 | 0 +(1 row) + SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IN (1, 2, 51, 52) AND b IN ( ''1'', ''2'')'); estimated | actual -----------+-------- @@ -1081,6 +1093,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' 200 | 200 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + estimated | actual +-----------+-------- + 3750 | 3750 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + estimated | actual +-----------+-------- + 1 | 0 +(1 row) + SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IN (1, 2, 51, 52) AND b IN ( ''1'', ''2'')'); estimated | actual -----------+-------- @@ -1284,6 +1308,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IS NULL AND 3750 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 25 | 2500 +(1 row) + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, d FROM mcv_lists; ANALYZE mcv_lists; @@ -1317,6 +1347,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IS NULL AND 2500 | 2500 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- mcv with pass-by-ref fixlen types, e.g. uuid CREATE TABLE mcv_lists_uuid ( a UUID, @@ -1408,6 +1444,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1094 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 9950 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 50 | 2500 +(1 row) + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; ANALYZE mcv_lists_bool; @@ -1435,6 +1483,18 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND 1 | 0 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + estimated | actual +-----------+-------- + 2500 | 2500 +(1 row) + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -1493,6 +1553,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR 2649 | 1572 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 1 | 5000 +(1 row) + -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; CREATE STATISTICS mcv_lists_multi_2 (mcv) ON c, d FROM mcv_lists_multi; @@ -1539,6 +1605,12 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR 1571 | 1572 (1 row) +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); + estimated | actual +-----------+-------- + 5000 | 5000 +(1 row) + DROP TABLE mcv_lists_multi; -- Permission tests. Users should not be able to see specific data values in -- the extended statistics, if they lack permission to see those values in diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql index 54ceb1f4ee..f5e4468129 100644 --- a/src/test/regress/sql/stats_ext.sql +++ b/src/test/regress/sql/stats_ext.sql @@ -498,6 +498,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IN (1, 2, 51, 52) AND b IN ( ''1'', ''2'')'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IN (1, 2, 51, 52, NULL) AND b IN ( ''1'', ''2'', NULL)'); @@ -549,6 +553,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ''1'' OR c = 1 OR d IS NOT NULL'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a > c'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a < c'); + SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IN (1, 2, 51, 52) AND b IN ( ''1'', ''2'')'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IN (1, 2, 51, 52, NULL) AND b IN ( ''1'', ''2'', NULL)'); @@ -656,6 +664,8 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IS NULL AND (b = ''x'' OR d = ''x'')'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- create statistics CREATE STATISTICS mcv_lists_stats (mcv) ON a, b, d FROM mcv_lists; @@ -674,6 +684,8 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a = 1 OR b = ' SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE a IS NULL AND (b = ''x'' OR d = ''x'')'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists WHERE b = d'); + -- mcv with pass-by-ref fixlen types, e.g. uuid CREATE TABLE mcv_lists_uuid ( a UUID, @@ -746,6 +758,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + CREATE STATISTICS mcv_lists_bool_stats (mcv) ON a, b, c FROM mcv_lists_bool; @@ -759,6 +775,10 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a AND b AND NOT c'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b'); + +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_bool WHERE NOT a = b AND b = c'); + -- check the ability to use multiple MCV lists CREATE TABLE mcv_lists_multi ( a INTEGER, @@ -785,6 +805,7 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE b = 0 OR SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE (a = 0 AND b = 0) OR (c = 0 AND d = 0)'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR b = 0 OR c = 0 OR d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); -- create separate MCV statistics CREATE STATISTICS mcv_lists_multi_1 (mcv) ON a, b FROM mcv_lists_multi; @@ -799,6 +820,7 @@ SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE b = 0 OR SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 AND b = 0 AND c = 0 AND d = 0'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE (a = 0 AND b = 0) OR (c = 0 AND d = 0)'); SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = 0 OR b = 0 OR c = 0 OR d = 0'); +SELECT * FROM check_estimated_rows('SELECT * FROM mcv_lists_multi WHERE a = b AND c = d'); DROP TABLE mcv_lists_multi; -- 2.21.1 --qmgrvggedyprimol-- ^ permalink raw reply [nested|flat] 6+ messages in thread
* [PATCH v49 6/7] Teach snapshot builder to skip transactions running REPACK (CONCURRENTLY). @ 2026-03-17 19:22 Antonin Houska <ah@cybertec.at> 0 siblings, 0 replies; 6+ messages in thread From: Antonin Houska @ 2026-03-17 19:22 UTC (permalink / raw) During logical decoding, we need to know if particular transaction performed catalog changes because catalog information is needed to construct heap tuples. To be sure that we have enough information of each transaction, the logical decoding cannot start before all the already running transactions have completed. The problem with REPACK (CONCURRENTLY) is that it has XID assigned and writes WAL records marked with it. Thus if another backend runs REPACK (CONCURRENTLY) and tries to setup the logical decoding, it has to wait for the completion of all the other transactions involved in REPACK (CONCURRENTLY). However, REPACK (CONCURRENTLY) does not perform any catalog changes relevant to logical decoding, so the other backends executing this command can ignore it. This patch implements it by adding information about transactions executing the command to the xl_running_xacts WAL record, and by teaching the snapshot builder to use the information. --- src/backend/access/rmgrdesc/standbydesc.c | 15 ++++- src/backend/access/transam/xlog.c | 2 + src/backend/commands/repack.c | 16 +++++ src/backend/replication/logical/snapbuild.c | 51 +++++++++----- src/backend/storage/ipc/procarray.c | 75 +++++++++++++++++++-- src/backend/storage/ipc/standby.c | 8 ++- src/include/storage/standby.h | 2 + src/include/storage/standbydefs.h | 2 + 8 files changed, 144 insertions(+), 27 deletions(-) diff --git a/src/backend/access/rmgrdesc/standbydesc.c b/src/backend/access/rmgrdesc/standbydesc.c index 0a291354ae2..58ebecf4927 100644 --- a/src/backend/access/rmgrdesc/standbydesc.c +++ b/src/backend/access/rmgrdesc/standbydesc.c @@ -21,10 +21,11 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec) { int i; - appendStringInfo(buf, "nextXid %u latestCompletedXid %u oldestRunningXid %u", + appendStringInfo(buf, "nextXid %u latestCompletedXid %u oldestRunningXid %u oldestRunningXidLogical %u", xlrec->nextXid, xlrec->latestCompletedXid, - xlrec->oldestRunningXid); + xlrec->oldestRunningXid, + xlrec->oldestRunningXidLogical); if (xlrec->xcnt > 0) { appendStringInfo(buf, "; %d xacts:", xlrec->xcnt); @@ -41,6 +42,16 @@ standby_desc_running_xacts(StringInfo buf, xl_running_xacts *xlrec) for (i = 0; i < xlrec->subxcnt; i++) appendStringInfo(buf, " %u", xlrec->xids[xlrec->xcnt + i]); } + + if (xlrec->xcnt_repack > 0) + { + TransactionId *xids_repack; + + appendStringInfo(buf, "; %d xacts_repack:", xlrec->xcnt_repack); + xids_repack = xlrec->xids + xlrec->xcnt + xlrec->subxcnt; + for (i = 0; i < xlrec->xcnt_repack; i++) + appendStringInfo(buf, " %u", xids_repack[i]); + } } void diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 2c1c6f88b74..f1cbdee7618 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5912,6 +5912,7 @@ StartupXLOG(void) * subxids are listed with their parent prepared transactions. */ running.xcnt = nxids; + running.xcnt_repack = 0; running.subxcnt = 0; running.subxid_status = SUBXIDS_IN_SUBTRANS; running.nextXid = XidFromFullTransactionId(checkPoint.nextXid); @@ -8483,6 +8484,7 @@ xlog_redo(XLogReaderState *record) * with their parent prepared transactions. */ running.xcnt = nxids; + running.xcnt_repack = 0; running.subxcnt = 0; running.subxid_status = SUBXIDS_IN_SUBTRANS; running.nextXid = XidFromFullTransactionId(checkPoint.nextXid); diff --git a/src/backend/commands/repack.c b/src/backend/commands/repack.c index 274ad900c65..5188244dfd6 100644 --- a/src/backend/commands/repack.c +++ b/src/backend/commands/repack.c @@ -993,6 +993,22 @@ rebuild_relation(Relation OldHeap, Relation index, bool verbose, if (concurrent) { + /* + * Do not let other backends wait for our completion during their + * setup of logical replication. Unlike logical replication publisher, + * we will have XID assigned, so the other backends - whether + * walsenders involved in logical replication or regular backends + * executing also REPACK (CONCURRENTLY) - would have to wait for our + * completion before they can build their initial snapshot. It is o.k. + * for any decoding backend to ignore us because we do not change + * tuple descriptor of any table, and the data changes we write should + * not be decoded by other backends. + */ + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + MyProc->statusFlags |= PROC_IN_CONCURRENT_REPACK; + ProcGlobal->statusFlags[MyProc->pgxactoff] = MyProc->statusFlags; + LWLockRelease(ProcArrayLock); + /* * The worker needs to be member of the locking group we're the leader * of. We ought to become the leader before the worker starts. The diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index 0f6b5df322c..23511a0905a 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -1172,7 +1172,7 @@ SnapBuildProcessRunningXacts(SnapBuild *builder, XLogRecPtr lsn, xl_running_xact * xmin, which looks odd but is correct and actually more efficient, since * we hit fast paths in heapam_visibility.c. */ - builder->xmin = running->oldestRunningXid; + builder->xmin = running->oldestRunningXidLogical; /* Remove transactions we don't need to keep track off anymore */ SnapBuildPurgeOlderTxn(builder); @@ -1188,9 +1188,9 @@ SnapBuildProcessRunningXacts(SnapBuild *builder, XLogRecPtr lsn, xl_running_xact */ xmin = ReorderBufferGetOldestXmin(builder->reorder); if (xmin == InvalidTransactionId) - xmin = running->oldestRunningXid; + xmin = running->oldestRunningXidLogical; elog(DEBUG3, "xmin: %u, xmax: %u, oldest running: %u, oldest xmin: %u", - builder->xmin, builder->xmax, running->oldestRunningXid, xmin); + builder->xmin, builder->xmax, running->oldestRunningXidLogical, xmin); LogicalIncreaseXminForSlot(lsn, xmin); /* @@ -1275,14 +1275,14 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn * have all necessary catalog rows anymore. */ if (TransactionIdIsNormal(builder->initial_xmin_horizon) && - NormalTransactionIdPrecedes(running->oldestRunningXid, + NormalTransactionIdPrecedes(running->oldestRunningXidLogical, builder->initial_xmin_horizon)) { ereport(DEBUG1, errmsg_internal("skipping snapshot at %X/%08X while building logical decoding snapshot, xmin horizon too low", LSN_FORMAT_ARGS(lsn)), errdetail_internal("initial xmin horizon of %u vs the snapshot's %u", - builder->initial_xmin_horizon, running->oldestRunningXid)); + builder->initial_xmin_horizon, running->oldestRunningXidLogical)); SnapBuildWaitSnapshot(running, builder->initial_xmin_horizon); @@ -1299,7 +1299,7 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn * NB: We might have already started to incrementally assemble a snapshot, * so we need to be careful to deal with that. */ - if (running->oldestRunningXid == running->nextXid) + if (running->oldestRunningXidLogical == running->nextXid) { if (!XLogRecPtrIsValid(builder->start_decoding_at) || builder->start_decoding_at <= lsn) @@ -1378,14 +1378,14 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn /* * c) transition from BUILDING_SNAPSHOT to FULL_SNAPSHOT. * - * In BUILDING_SNAPSHOT state, and this xl_running_xacts' oldestRunningXid - * is >= than nextXid from when we switched to BUILDING_SNAPSHOT. This - * means all transactions starting afterwards have enough information to - * be decoded. Switch to FULL_SNAPSHOT. + * In BUILDING_SNAPSHOT state, and this xl_running_xacts' + * oldestRunningXidLogical is >= than nextXid from when we switched to + * BUILDING_SNAPSHOT. This means all transactions starting afterwards + * have enough information to be decoded. Switch to FULL_SNAPSHOT. */ else if (builder->state == SNAPBUILD_BUILDING_SNAPSHOT && TransactionIdPrecedesOrEquals(builder->next_phase_at, - running->oldestRunningXid)) + running->oldestRunningXidLogical)) { builder->state = SNAPBUILD_FULL_SNAPSHOT; builder->next_phase_at = running->nextXid; @@ -1402,14 +1402,15 @@ SnapBuildFindSnapshot(SnapBuild *builder, XLogRecPtr lsn, xl_running_xacts *runn /* * c) transition from FULL_SNAPSHOT to CONSISTENT. * - * In FULL_SNAPSHOT state, and this xl_running_xacts' oldestRunningXid is - * >= than nextXid from when we switched to FULL_SNAPSHOT. This means all - * transactions that are currently in progress have a catalog snapshot, - * and all their changes have been collected. Switch to CONSISTENT. + * In FULL_SNAPSHOT state, and this xl_running_xacts' + * oldestRunningXidLogical is >= than nextXid from when we switched to + * FULL_SNAPSHOT. This means all transactions that are currently in + * progress have a catalog snapshot, and all their changes have been + * collected. Switch to CONSISTENT. */ else if (builder->state == SNAPBUILD_FULL_SNAPSHOT && TransactionIdPrecedesOrEquals(builder->next_phase_at, - running->oldestRunningXid)) + running->oldestRunningXidLogical)) { builder->state = SNAPBUILD_CONSISTENT; builder->next_phase_at = InvalidTransactionId; @@ -1459,6 +1460,24 @@ SnapBuildWaitSnapshot(xl_running_xacts *running, TransactionId cutoff) if (TransactionIdFollows(xid, cutoff)) continue; + /* Do not wait for transactions running REPACK (CONCURRENTLY). */ + if (running->xcnt_repack > 0) + { + TransactionId *xids_repack; + int i; + + xids_repack = running->xids + running->xcnt + running->subxcnt; + + for (i = 0; i < running->xcnt_repack; i++) + { + if (xid == xids_repack[i]) + break; + } + /* Found? */ + if (i < running->xcnt_repack) + continue; + } + XactLockTableWait(xid, NULL, NULL, XLTW_None); } diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index cc207cb56e3..033ae68b57e 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -2643,15 +2643,25 @@ GetRunningTransactionData(void) RunningTransactions CurrentRunningXacts = &CurrentRunningXactsData; TransactionId latestCompletedXid; TransactionId oldestRunningXid; + TransactionId oldestRunningXidLogical; TransactionId oldestDatabaseRunningXid; TransactionId *xids; int index; - int count; + int count, + count_repack; int subcount; bool suboverflowed; + TransactionId *xids_repack = NULL; + bool logical_decoding_enabled = IsLogicalDecodingEnabled(); Assert(!RecoveryInProgress()); + /* + * TODO Consider a GUC to reserve certain amount of replication slots for + * REPACK (CONCURRENTLY) and using it here. + */ +#define MAX_REPACK_XIDS 16 + /* * Allocating space for maxProcs xids is usually overkill; numProcs would * be sufficient. But it seems better to do the malloc while not holding @@ -2663,11 +2673,14 @@ GetRunningTransactionData(void) */ if (CurrentRunningXacts->xids == NULL) { + /* FIXME probably fails if logical decoding is enable on-the-fly */ + int nrepack = logical_decoding_enabled ? MAX_REPACK_XIDS : 0; + /* * First call */ CurrentRunningXacts->xids = (TransactionId *) - malloc(TOTAL_MAX_CACHED_SUBXIDS * sizeof(TransactionId)); + malloc((TOTAL_MAX_CACHED_SUBXIDS + nrepack) * sizeof(TransactionId)); if (CurrentRunningXacts->xids == NULL) ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), @@ -2676,7 +2689,10 @@ GetRunningTransactionData(void) xids = CurrentRunningXacts->xids; - count = subcount = 0; + if (logical_decoding_enabled) + xids_repack = palloc_array(TransactionId, MAX_REPACK_XIDS); + + count = subcount = count_repack = 0; suboverflowed = false; /* @@ -2688,7 +2704,7 @@ GetRunningTransactionData(void) latestCompletedXid = XidFromFullTransactionId(TransamVariables->latestCompletedXid); - oldestDatabaseRunningXid = oldestRunningXid = + oldestDatabaseRunningXid = oldestRunningXid = oldestRunningXidLogical = XidFromFullTransactionId(TransamVariables->nextXid); /* @@ -2697,6 +2713,8 @@ GetRunningTransactionData(void) for (index = 0; index < arrayP->numProcs; index++) { TransactionId xid; + int pgprocno; + PGPROC *proc; /* Fetch xid just once - see GetNewTransactionId */ xid = UINT32_ACCESS_ONCE(other_xids[index]); @@ -2716,6 +2734,21 @@ GetRunningTransactionData(void) if (TransactionIdPrecedes(xid, oldestRunningXid)) oldestRunningXid = xid; + if (logical_decoding_enabled && + TransactionIdPrecedes(xid, oldestRunningXidLogical)) + { + /* + * Backends running REPACK concurrently need to be excluded from + * oldestRunningXidLogical, otherwise the snapshot builder cannot + * proceed in building the initial snapshot. + */ + pgprocno = arrayP->pgprocnos[index]; + proc = &allProcs[pgprocno]; + + if ((proc->statusFlags & PROC_IN_CONCURRENT_REPACK) == 0) + oldestRunningXidLogical = xid; + } + /* * Also, update the oldest running xid within the current database. As * fetching pgprocno and PGPROC could cause cache misses, we do cheap @@ -2723,8 +2756,8 @@ GetRunningTransactionData(void) */ if (TransactionIdPrecedes(xid, oldestDatabaseRunningXid)) { - int pgprocno = arrayP->pgprocnos[index]; - PGPROC *proc = &allProcs[pgprocno]; + pgprocno = arrayP->pgprocnos[index]; + proc = &allProcs[pgprocno]; if (proc->databaseId == MyDatabaseId) oldestDatabaseRunningXid = xid; @@ -2742,6 +2775,19 @@ GetRunningTransactionData(void) */ xids[count++] = xid; + + /* + * Collect XIDSs of transactions running REPACK (CONCURRENTLY). + */ + if (logical_decoding_enabled && + count_repack < MAX_REPACK_XIDS) + { + pgprocno = arrayP->pgprocnos[index]; + proc = &allProcs[pgprocno]; + + if (proc->statusFlags & PROC_IN_CONCURRENT_REPACK) + xids_repack[count_repack++] = xid; + } } /* @@ -2782,6 +2828,19 @@ GetRunningTransactionData(void) } } + /* + * Append the XIDs running REPACK (CONCURRENTLY), if any. + * + * XXX Should we sort the array and use bsearch() when using it? + */ + if (count_repack > 0) + { + for (int i = 0; i < count_repack; i++) + xids[count++] = xids_repack[i]; + } + if (xids_repack) + pfree(xids_repack); + /* * It's important *not* to include the limits set by slots here because * snapbuild.c uses oldestRunningXid to manage its xmin horizon. If those @@ -2791,11 +2850,13 @@ GetRunningTransactionData(void) * increases if slots do. */ - CurrentRunningXacts->xcnt = count - subcount; + CurrentRunningXacts->xcnt = count - subcount - count_repack; CurrentRunningXacts->subxcnt = subcount; + CurrentRunningXacts->xcnt_repack = count_repack; CurrentRunningXacts->subxid_status = suboverflowed ? SUBXIDS_IN_SUBTRANS : SUBXIDS_IN_ARRAY; CurrentRunningXacts->nextXid = XidFromFullTransactionId(TransamVariables->nextXid); CurrentRunningXacts->oldestRunningXid = oldestRunningXid; + CurrentRunningXacts->oldestRunningXidLogical = oldestRunningXidLogical; CurrentRunningXacts->oldestDatabaseRunningXid = oldestDatabaseRunningXid; CurrentRunningXacts->latestCompletedXid = latestCompletedXid; diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c index de9092fdf5b..fd3e4fe31f3 100644 --- a/src/backend/storage/ipc/standby.c +++ b/src/backend/storage/ipc/standby.c @@ -1189,6 +1189,7 @@ standby_redo(XLogReaderState *record) RunningTransactionsData running; running.xcnt = xlrec->xcnt; + running.xcnt_repack = xlrec->xcnt_repack; running.subxcnt = xlrec->subxcnt; running.subxid_status = xlrec->subxid_overflow ? SUBXIDS_MISSING : SUBXIDS_IN_ARRAY; running.nextXid = xlrec->nextXid; @@ -1359,10 +1360,12 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts) XLogRecPtr recptr; xlrec.xcnt = CurrRunningXacts->xcnt; + xlrec.xcnt_repack = CurrRunningXacts->xcnt_repack; xlrec.subxcnt = CurrRunningXacts->subxcnt; xlrec.subxid_overflow = (CurrRunningXacts->subxid_status != SUBXIDS_IN_ARRAY); xlrec.nextXid = CurrRunningXacts->nextXid; xlrec.oldestRunningXid = CurrRunningXacts->oldestRunningXid; + xlrec.oldestRunningXidLogical = CurrRunningXacts->oldestRunningXidLogical; xlrec.latestCompletedXid = CurrRunningXacts->latestCompletedXid; /* Header */ @@ -1371,9 +1374,10 @@ LogCurrentRunningXacts(RunningTransactions CurrRunningXacts) XLogRegisterData(&xlrec, MinSizeOfXactRunningXacts); /* array of TransactionIds */ - if (xlrec.xcnt > 0) + if (xlrec.xcnt + xlrec.xcnt_repack > 0) XLogRegisterData(CurrRunningXacts->xids, - (xlrec.xcnt + xlrec.subxcnt) * sizeof(TransactionId)); + (xlrec.xcnt + xlrec.xcnt_repack + xlrec.subxcnt) * + sizeof(TransactionId)); recptr = XLogInsert(RM_STANDBY_ID, XLOG_RUNNING_XACTS); diff --git a/src/include/storage/standby.h b/src/include/storage/standby.h index 6a314c693cd..fe4dcdc3def 100644 --- a/src/include/storage/standby.h +++ b/src/include/storage/standby.h @@ -127,10 +127,12 @@ typedef enum typedef struct RunningTransactionsData { int xcnt; /* # of xact ids in xids[] */ + int xcnt_repack; /* # of xacts running REPACK (CONCURRENTLY). */ int subxcnt; /* # of subxact ids in xids[] */ subxids_array_status subxid_status; TransactionId nextXid; /* xid from TransamVariables->nextXid */ TransactionId oldestRunningXid; /* *not* oldestXmin */ + TransactionId oldestRunningXidLogical; TransactionId oldestDatabaseRunningXid; /* same as above, but within the * current database */ TransactionId latestCompletedXid; /* so we can set xmax */ diff --git a/src/include/storage/standbydefs.h b/src/include/storage/standbydefs.h index 231d251fd51..edad609fa9a 100644 --- a/src/include/storage/standbydefs.h +++ b/src/include/storage/standbydefs.h @@ -47,10 +47,12 @@ typedef struct xl_standby_locks typedef struct xl_running_xacts { int xcnt; /* # of xact ids in xids[] */ + int xcnt_repack; /* # of xacts running REPACK (CONCURRENTLY) */ int subxcnt; /* # of subxact ids in xids[] */ bool subxid_overflow; /* snapshot overflowed, subxids missing */ TransactionId nextXid; /* xid from TransamVariables->nextXid */ TransactionId oldestRunningXid; /* *not* oldestXmin */ + TransactionId oldestRunningXidLogical; TransactionId latestCompletedXid; /* so we can set xmax */ TransactionId xids[FLEXIBLE_ARRAY_MEMBER]; -- 2.47.3 --3n3vqr5y2jhknrqj Content-Type: text/x-diff; charset=utf-8 Content-Disposition: attachment; filename="v49-0007-WIP-add-max_repack_replication_slots.patch" ^ permalink raw reply [nested|flat] 6+ messages in thread
end of thread, other threads:[~2026-03-17 19:22 UTC | newest] Thread overview: 6+ messages (download: mbox mbox.gz follow: Atom feed) -- links below jump to the message on this page -- 2019-11-11 00:34 [PATCH 2/3] Support clauses of the form Var op Var Tomas Vondra <tv@fuzzy.cz> 2019-11-11 00:34 [PATCH 2/3] Support clauses of the form Var op Var Tomas Vondra <tv@fuzzy.cz> 2019-11-11 00:34 [PATCH 2/3] Support clauses of the form Var op Var Tomas Vondra <tv@fuzzy.cz> 2020-03-08 22:27 [PATCH 4/4] Support clauses of the form Var op Var Tomas Vondra <tv@fuzzy.cz> 2020-03-17 01:57 [PATCH 2/2] Support clauses of the form Var op Var Tomas Vondra <tv@fuzzy.cz> 2026-03-17 19:22 [PATCH v49 6/7] Teach snapshot builder to skip transactions running REPACK (CONCURRENTLY). Antonin Houska <ah@cybertec.at>
This inbox is served by agora; see mirroring instructions for how to clone and mirror all data and code used for this inbox