From 70cf8ed9f0d161a335be1e72a25f325261ea9e46 Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Mon, 6 Apr 2015 19:42:18 +0200 Subject: [PATCH 3/9] clause reduction using functional dependencies During planning, use functional dependencies to decide which clauses to skip during cardinality estimation. Initial and rather simplistic implementation. This only works with regular WHERE clauses, not clauses used for join clauses. Note: The clause_is_mv_compatible() needs to identify the relation (so that we can fetch the list of multivariate stats by OID). planner_rt_fetch() seems like the appropriate way to get the relation OID, but apparently it only works with simple vars. Maybe examine_variable() would make this work with more complex vars too? Includes regression tests analyzing functional dependencies (part of ANALYZE) on several datasets (no dependencies, no transitive dependencies, ...). Checks that a query with conditions on two columns, where one (B) is functionally dependent on the other one (A), correctly ignores the clause on (B) and chooses bitmap index scan instead of plain index scan (which is what happens otherwise, thanks to assumption of independence). Note: Functional dependencies only work with equality clauses, no inequalities etc. --- src/backend/optimizer/path/clausesel.c | 505 ++++++++++++++++- src/backend/utils/mvstats/README.dependencies | 63 +-- src/backend/utils/mvstats/README.stats | 36 ++ src/backend/utils/mvstats/common.c | 25 +- src/backend/utils/mvstats/common.h | 3 + src/backend/utils/mvstats/dependencies.c | 775 +++++++++++++++++--------- src/include/utils/mvstats.h | 21 +- src/test/regress/expected/mv_dependencies.out | 172 ++++++ src/test/regress/parallel_schedule | 3 + src/test/regress/serial_schedule | 1 + src/test/regress/sql/mv_dependencies.sql | 150 +++++ 11 files changed, 1457 insertions(+), 297 deletions(-) create mode 100644 src/backend/utils/mvstats/README.stats create mode 100644 src/test/regress/expected/mv_dependencies.out create mode 100644 src/test/regress/sql/mv_dependencies.sql diff --git a/src/backend/optimizer/path/clausesel.c b/src/backend/optimizer/path/clausesel.c index 02660c2..5ab7f15 100644 --- a/src/backend/optimizer/path/clausesel.c +++ b/src/backend/optimizer/path/clausesel.c @@ -14,14 +14,19 @@ */ #include "postgres.h" +#include "access/sysattr.h" +#include "catalog/pg_operator.h" #include "nodes/makefuncs.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" #include "optimizer/pathnode.h" #include "optimizer/plancat.h" +#include "optimizer/var.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" +#include "utils/mvstats.h" #include "utils/selfuncs.h" +#include "utils/typcache.h" /* @@ -41,6 +46,25 @@ typedef struct RangeQueryClause static void addRangeClause(RangeQueryClause **rqlist, Node *clause, bool varonleft, bool isLTsel, Selectivity s2); +#define MV_CLAUSE_TYPE_FDEP 0x01 + +static bool clause_is_mv_compatible(Node *clause, Index relid, AttrNumber *attnum); + +static Bitmapset *collect_mv_attnums(List *clauses, Index relid); + +static int count_mv_attnums(List *clauses, Index relid); + +static int count_varnos(List *clauses, Index *relid); + +static List *clauselist_apply_dependencies(PlannerInfo *root, List *clauses, + Index relid, List *stats); + +static bool has_stats(List *stats, int type); + +static List * find_stats(PlannerInfo *root, Index relid); + +static bool stats_type_matches(MVStatisticInfo *stat, int type); + /**************************************************************************** * ROUTINES TO COMPUTE SELECTIVITIES @@ -60,7 +84,19 @@ static void addRangeClause(RangeQueryClause **rqlist, Node *clause, * subclauses. However, that's only right if the subclauses have independent * probabilities, and in reality they are often NOT independent. So, * we want to be smarter where we can. - + * + * The first thing we try to do is applying multivariate statistics, in a way + * that intends to minimize the overhead when there are no multivariate stats + * on the relation. Thus we do several simple (and inexpensive) checks first, + * to verify that suitable multivariate statistics exist. + * + * If we identify such multivariate statistics apply, we try to apply them. + * Currently we only have (soft) functional dependencies, so we try to reduce + * the list of clauses. + * + * Then we remove the clauses estimated using multivariate stats, and process + * the rest of the clauses using the regular per-column stats. + * * Currently, the only extra smarts we have is to recognize "range queries", * such as "x > 34 AND x < 42". Clauses are recognized as possible range * query components if they are restriction opclauses whose operators have @@ -99,6 +135,22 @@ clauselist_selectivity(PlannerInfo *root, RangeQueryClause *rqlist = NULL; ListCell *l; + /* processing mv stats */ + Oid relid = InvalidOid; + + /* list of multivariate stats on the relation */ + List *stats = NIL; + + /* + * To fetch the statistics, we first need to determine the rel. Currently + * point we only support estimates of simple restrictions with all Vars + * referencing a single baserel. However set_baserel_size_estimates() sets + * varRelid=0 so we have to actually inspect the clauses by pull_varnos + * and see if there's just a single varno referenced. + */ + if ((count_varnos(clauses, &relid) == 1) && ((varRelid == 0) || (varRelid == relid))) + stats = find_stats(root, relid); + /* * If there's exactly one clause, then no use in trying to match up pairs, * so just go directly to clause_selectivity(). @@ -108,6 +160,24 @@ clauselist_selectivity(PlannerInfo *root, varRelid, jointype, sjinfo); /* + * Apply functional dependencies, but first check that there are some stats + * with functional dependencies built (by simply walking the stats list), + * and that there are at two or more attributes referenced by clauses that + * may be reduced using functional dependencies. + * + * We would find that anyway when trying to actually apply the functional + * dependencies, but let's do the cheap checks first. + * + * After applying the functional dependencies we get the remainig clauses + * that need to be estimated by other types of stats (MCV, histograms etc). + */ + if (has_stats(stats, MV_CLAUSE_TYPE_FDEP) && + (count_mv_attnums(clauses, relid) >= 2)) + { + clauses = clauselist_apply_dependencies(root, clauses, relid, stats); + } + + /* * Initial scan over clauses. Anything that doesn't look like a potential * rangequery clause gets multiplied into s1 and forgotten. Anything that * does gets inserted into an rqlist entry. @@ -763,3 +833,436 @@ clause_selectivity(PlannerInfo *root, return s1; } + +/* + * Collect attributes from mv-compatible clauses. + */ +static Bitmapset * +collect_mv_attnums(List *clauses, Index relid) +{ + Bitmapset *attnums = NULL; + ListCell *l; + + /* + * Walk through the clauses and identify the ones we can estimate + * using multivariate stats, and remember the relid/columns. We'll + * then cross-check if we have suitable stats, and only if needed + * we'll split the clauses into multivariate and regular lists. + * + * For now we're only interested in RestrictInfo nodes with nested + * OpExpr, using either a range or equality. + */ + foreach (l, clauses) + { + AttrNumber attnum; + Node *clause = (Node *) lfirst(l); + + /* ignore the result for now - we only need the info */ + if (clause_is_mv_compatible(clause, relid, &attnum)) + attnums = bms_add_member(attnums, attnum); + } + + /* + * If there are not at least two attributes referenced by the clause(s), + * we can throw everything out (as we'll revert to simple stats). + */ + if (bms_num_members(attnums) <= 1) + { + if (attnums != NULL) + pfree(attnums); + attnums = NULL; + } + + return attnums; +} + +/* + * Count the number of attributes in clauses compatible with multivariate stats. + */ +static int +count_mv_attnums(List *clauses, Index relid) +{ + int c; + Bitmapset *attnums = collect_mv_attnums(clauses, relid); + + c = bms_num_members(attnums); + + bms_free(attnums); + + return c; +} + +/* + * Count varnos referenced in the clauses, and if there's a single varno then + * return the index in 'relid'. + */ +static int +count_varnos(List *clauses, Index *relid) +{ + int cnt; + Bitmapset *varnos = NULL; + + varnos = pull_varnos((Node *) clauses); + cnt = bms_num_members(varnos); + + /* if there's a single varno in the clauses, remember it */ + if (bms_num_members(varnos) == 1) + *relid = bms_singleton_member(varnos); + + bms_free(varnos); + + return cnt; +} + +typedef struct +{ + Index varno; /* relid we're interested in */ + Bitmapset *varattnos; /* attnums referenced by the clauses */ +} mv_compatible_context; + +/* + * Recursive walker that checks compatibility of the clause with multivariate + * statistics, and collects attnums from the Vars. + * + * XXX The original idea was to combine this with expression_tree_walker, but + * I've been unable to make that work - seems that does not quite allow + * checking the structure. Hence the explicit calls to the walker. + */ +static bool +mv_compatible_walker(Node *node, mv_compatible_context *context) +{ + if (node == NULL) + return false; + + if (IsA(node, RestrictInfo)) + { + RestrictInfo *rinfo = (RestrictInfo *) node; + + /* Pseudoconstants are not really interesting here. */ + if (rinfo->pseudoconstant) + return true; + + /* clauses referencing multiple varnos are incompatible */ + if (bms_membership(rinfo->clause_relids) != BMS_SINGLETON) + return true; + + /* check the clause inside the RestrictInfo */ + return mv_compatible_walker((Node*)rinfo->clause, (void *) context); + } + + if (IsA(node, Var)) + { + Var * var = (Var*)node; + + /* + * Also, the variable needs to reference the right relid (this might be + * unnecessary given the other checks, but let's be sure). + */ + if (var->varno != context->varno) + return true; + + /* Also skip system attributes (we don't allow stats on those). */ + if (! AttrNumberIsForUserDefinedAttr(var->varattno)) + return true; + + /* Seems fine, so let's remember the attnum. */ + context->varattnos = bms_add_member(context->varattnos, var->varattno); + + return false; + } + + /* + * And finally the operator expressions - we only allow simple expressions + * with two arguments, where one is a Var and the other is a constant, and + * it's a simple comparison (which we detect using estimator function). + */ + if (is_opclause(node)) + { + OpExpr *expr = (OpExpr *) node; + Var *var; + bool varonleft = true; + bool ok; + + /* + * Only expressions with two arguments are considered compatible. + * + * XXX Possibly unnecessary (can OpExpr have different arg count?). + */ + if (list_length(expr->args) != 2) + return true; + + /* see if it actually has the right */ + ok = (NumRelids((Node*)expr) == 1) && + (is_pseudo_constant_clause(lsecond(expr->args)) || + (varonleft = false, + is_pseudo_constant_clause(linitial(expr->args)))); + + /* unsupported structure (two variables or so) */ + if (! ok) + return true; + + /* + * If it's not a "<" or ">" or "=" operator, just ignore the clause. + * Otherwise note the relid and attnum for the variable. This uses the + * function for estimating selectivity, ont the operator directly (a bit + * awkward, but well ...). + */ + switch (get_oprrest(expr->opno)) + { + case F_EQSEL: + + /* equality conditions are compatible with all statistics */ + break; + + default: + + /* unknown estimator */ + return true; + } + + var = (varonleft) ? linitial(expr->args) : lsecond(expr->args); + + return mv_compatible_walker((Node *) var, context); + } + + /* Node not explicitly supported, so terminate */ + return true; +} + +/* + * Determines whether the clause is compatible with multivariate stats, + * and if it is, returns some additional information - varno (index + * into simple_rte_array) and a bitmap of attributes. This is then + * used to fetch related multivariate statistics. + * + * At this moment we only support basic conditions of the form + * + * variable OP constant + * + * where OP is one of [=,<,<=,>=,>] (which is however determined by + * looking at the associated function for estimating selectivity, just + * like with the single-dimensional case). + * + * TODO Support 'OR clauses' - shouldn't be all that difficult to + * evaluate them using multivariate stats. + */ +static bool +clause_is_mv_compatible(Node *clause, Index relid, AttrNumber *attnum) +{ + mv_compatible_context context; + + context.varno = relid; + context.varattnos = NULL; /* no attnums */ + + if (mv_compatible_walker(clause, (void *) &context)) + return false; + + /* remember the newly collected attnums */ + *attnum = bms_singleton_member(context.varattnos); + + return true; +} + + +/* + * Reduce clauses using functional dependencies + */ +static List* +fdeps_reduce_clauses(List *clauses, Index relid, Bitmapset *reduced_attnums) +{ + ListCell *lc; + List *reduced_clauses = NIL; + + foreach (lc, clauses) + { + AttrNumber attnum = InvalidAttrNumber; + Node * clause = (Node*)lfirst(lc); + + /* ignore clauses that are not compatible with functional dependencies */ + if (! clause_is_mv_compatible(clause, relid, &attnum)) + reduced_clauses = lappend(reduced_clauses, clause); + + /* for equality clauses, only keep those not on reduced attributes */ + if (! bms_is_member(attnum, reduced_attnums)) + reduced_clauses = lappend(reduced_clauses, clause); + } + + return reduced_clauses; +} + +/* + * decide which attributes are redundant (for equality clauses) + * + * We try to apply all functional dependencies available, and for each one we + * check if it matches attnums from equality clauses, but only those not yet + * reduced. + * + * XXX Not sure if the order in which we apply the dependencies matters. + * + * XXX We do not combine functional dependencies from separate stats. That is + * if we have dependencies on [a,b] and [b,c], then we don't deduce + * a->c from a->b and b->c. Computing such transitive closure is a possible + * future improvement. + */ +static Bitmapset * +fdeps_reduce_attnums(List *stats, Bitmapset *attnums) +{ + ListCell *lc; + Bitmapset *reduced = NULL; + + foreach (lc, stats) + { + int i; + MVDependencies dependencies = NULL; + MVStatisticInfo *info = (MVStatisticInfo *)lfirst(lc); + + /* skip statistics without dependencies */ + if (! stats_type_matches(info, MV_CLAUSE_TYPE_FDEP)) + continue; + + /* fetch and deserialize dependencies */ + dependencies = load_mv_dependencies(info->mvoid); + + for (i = 0; i < dependencies->ndeps; i++) + { + int j; + bool matched = true; + MVDependency dep = dependencies->deps[i]; + + /* we don't bother to break the loop early (only few attributes) */ + for (j = 0; j < dep->nattributes; j++) + { + if (! bms_is_member(dep->attributes[j], attnums)) + matched = false; + + if (bms_is_member(dep->attributes[j], reduced)) + matched = false; + } + + /* if dependency applies, mark the last attribute as reduced */ + if (matched) + reduced = bms_add_member(reduced, + dep->attributes[dep->nattributes-1]); + } + } + + return reduced; +} + +/* + * reduce list of equality clauses using soft functional dependencies + * + * We simply walk through list of functional dependencies, and for each one we + * check whether the dependency 'matches' the clauses, i.e. if there's a clause + * matching the condition. If yes, we attempt to remove all clauses matching + * the implied part of the dependency from the list. + * + * This only reduces equality clauses, and ignores all the other types. We might + * extend it to handle IS NULL clause, in the future. + * + * We also assume the equality clauses are 'compatible'. For example we can't + * identify when the clauses use a mismatching zip code and city name. In such + * case the usual approach (product of selectivities) would produce a better + * estimate, although mostly by chance. + * + * The implementation needs to be careful about cyclic dependencies, e.g. when + * + * (a -> b) and (b -> a) + * + * at the same time, which means there's 1:1 relationship between te columns. + * In this case we must not reduce clauses on both attributes at the same time. + * + * TODO Currently we only apply functional dependencies at the same level, but + * maybe we could transfer the clauses from upper levels to the subtrees? + * For example let's say we have (a->b) dependency, and condition + * + * (a=1) AND (b=2 OR c=3) + * + * Currently, we won't be able to perform any reduction, because we'll + * consider (a=1) and (b=2 OR c=3) independently. But maybe we could pass + * (a=1) into the other expression, and only check it against conditions + * of the functional dependencies? + * + * In this case we'd end up with + * + * (a=1) + * + * as we'd consider (b=2) implied thanks to the rule, rendering the whole + * OR clause valid. + */ +static List * +clauselist_apply_dependencies(PlannerInfo *root, List *clauses, + Index relid, List *stats) +{ + Bitmapset *clause_attnums = NULL; + Bitmapset *reduced_attnums = NULL; + + /* + * Is there at least one statistics with functional dependencies? + * If not, return the original clauses right away. + * + * XXX Isn't this a bit pointless, thanks to exactly the same check in + * clauselist_selectivity()? Can we trigger the condition here? + */ + if (! has_stats(stats, MV_CLAUSE_TYPE_FDEP)) + return clauses; + + /* collect attnums from clauses compatible with dependencies (equality) */ + clause_attnums = collect_mv_attnums(clauses, relid); + + /* decide which attnums may be eliminated */ + reduced_attnums = fdeps_reduce_attnums(stats, clause_attnums); + + /* + * Walk through the clauses, and see which other clauses we may reduce. + */ + clauses = fdeps_reduce_clauses(clauses, relid, reduced_attnums); + + bms_free(clause_attnums); + bms_free(reduced_attnums); + + return clauses; +} + +/* + * Check that there are stats with at least one of the requested types. + */ +static bool +stats_type_matches(MVStatisticInfo *stat, int type) +{ + if ((type & MV_CLAUSE_TYPE_FDEP) && stat->deps_built) + return true; + + return false; +} + +/* + * Check that there are stats with at least one of the requested types. + */ +static bool +has_stats(List *stats, int type) +{ + ListCell *s; + + foreach (s, stats) + { + MVStatisticInfo *stat = (MVStatisticInfo *)lfirst(s); + + /* terminate if we've found at least one matching statistics */ + if (stats_type_matches(stat, type)) + return true; + } + + return false; +} + +/* + * Lookups stats for a given baserel. + */ +static List * +find_stats(PlannerInfo *root, Index relid) +{ + Assert(root->simple_rel_array[relid] != NULL); + + return root->simple_rel_array[relid]->mvstatlist; +} diff --git a/src/backend/utils/mvstats/README.dependencies b/src/backend/utils/mvstats/README.dependencies index 1f96fbc..f248459 100644 --- a/src/backend/utils/mvstats/README.dependencies +++ b/src/backend/utils/mvstats/README.dependencies @@ -156,37 +156,24 @@ estimates - especially compared to histograms, that are quite bad in estimating equality clauses. -Limitations ------------ - -Let's see the main liminations of functional dependencies, especially those -related to the current implementation. +Multi-column dependencies +------------------------- -The current implementation supports only dependencies between two columns, but -this is merely a simplification of the initial implementation. It's certainly -useful to mine for dependencies involving multiple columns on the 'left' side, -i.e. a condition for the dependency. That is dependencies like (a,b -> c). +The implementation supports dependencies with multiple columns on the left side +(i.e. condition of the dependency). The detection starts from dependencies with +a single condition, and then proceeds to higher condition counts. -The implementation may/should be smart enough not to mine redundant conditions, -e.g. (a->b) and (a,c -> b), because the latter is a trivial consequence of the -former one (if values of 'a' determine 'b', adding another column won't change -that relationship). The ANALYZE should first analyze 1:1 dependencies, then 2:1 -dependencies (and skip the already identified ones), etc. +It also detects dependencies that are implied by already identified ones, and +ignores them. For example if we know that (a->b) then we won't add (a,c->b) as +this dependency is a trivial consequence of (a->b). -For example the dependency +For a more practical example, consider these two dependencies (city name -> zip code) - -is much stronger, i.e. whenever it hold, then - (city name, state name -> zip code) -holds too. But in case there are cities with the same name in different states, -then only the latter dependency will be valid. - -Of course, there probably are cities with the same name within a single state, -but hopefully this is relatively rare occurence (and thus we'll still detect -the 'soft' dependency). +We could say that the former dependency is stronger because if it's valid, then +the second dependency is valid too. Handling multiple columns on the right side of the dependency, is not necessary, as those dependencies may be simply decomposed into a set of dependencies with @@ -199,24 +186,22 @@ is exactly the same as (a -> b) & (a -> c) Of course, storing the first form may be more efficient thant storing multiple -'simple' dependencies separately. - +'simple' dependencies separately. This is left as a future work. -TODO Support dependencies with multiple columns on left/right. -TODO Investigate using histogram and MCV list to verify the dependencies. +Future work +----------- -TODO Investigate statistical testing of the distribution (to decide whether it - makes sense to build the histogram/MCV list). +* Investigate using histogram and MCV list to verify the dependencies. -TODO Using a min/max of selectivities would probably make more sense for the - associated columns. +* Investigate statistical testing of the distribution (to decide whether it + makes sense to build the histogram/MCV list). -TODO Consider eliminating the implied columns from the histogram and MCV lists - (but maybe that's not a good idea, because that'd make it impossible to use - these stats for non-equality clauses and also it wouldn't be possible to - use the stats for verification of the dependencies). +* Consider eliminating the implied columns from the histogram and MCV lists + (but maybe that's not a good idea, because that'd make it impossible to use + these stats for non-equality clauses and also it wouldn't be possible to + use the stats for verification of the dependencies). -TODO The reduction probably might be extended to also handle IS NULL clauses, - assuming we fix the ANALYZE to properly handle NULL values. We however - won't be able to reduce IS NOT NULL (unless I'm missing something). +* The reduction probably might be extended to also handle IS NULL clauses, + assuming we fix the ANALYZE to properly handle NULL values. We however + won't be able to reduce IS NOT NULL (unless I'm missing something). diff --git a/src/backend/utils/mvstats/README.stats b/src/backend/utils/mvstats/README.stats new file mode 100644 index 0000000..a38ea7b --- /dev/null +++ b/src/backend/utils/mvstats/README.stats @@ -0,0 +1,36 @@ +Multivariate statististics +========================== + +When estimating various quantities (e.g. condition selectivities) the default +approach relies on the assumption of independence. In practice that's often +not true, resulting in estimation errors. + +Multivariate stats track different types of dependencies between the columns, +hopefully improving the estimates. + +Currently we only have one kind of multivariate statistics - soft functional +dependencies, and we use it to improve estimates of equality clauses. See +README.dependencies for details. + + +Selectivity estimation +---------------------- + +When estimating selectivity, we aim to achieve several things: + + (a) maximize the estimate accuracy + + (b) minimize the overhead, especially when no suitable multivariate stats + exist (so if you are not using multivariate stats, there's no overhead) + +This clauselist_selectivity() performs several inexpensive checks first, before +even attempting to do the more expensive estimation. + + (1) check if there are multivariate stats on the relation + + (2) check there are at least two attributes referenced by clauses compatible + with multivariate statistics (equality clauses for func. dependencies) + + (3) perform reduction of equality clauses using func. dependencies + + (4) estimate the reduced list of clauses using regular statistics diff --git a/src/backend/utils/mvstats/common.c b/src/backend/utils/mvstats/common.c index a755c49..dcb7c78 100644 --- a/src/backend/utils/mvstats/common.c +++ b/src/backend/utils/mvstats/common.c @@ -84,7 +84,8 @@ build_mv_stats(Relation onerel, int numrows, HeapTuple *rows, /* * Analyze functional dependencies of columns. */ - deps = build_mv_dependencies(numrows, rows, attrs, stats); + if (stat->deps_enabled) + deps = build_mv_dependencies(numrows, rows, attrs, stats); /* store the histogram / MCV list in the catalog */ update_mv_stats(stat->mvoid, deps, attrs); @@ -163,6 +164,7 @@ list_mv_stats(Oid relid) info->mvoid = HeapTupleGetOid(htup); info->stakeys = buildint2vector(stats->stakeys.values, stats->stakeys.dim1); + info->deps_enabled = stats->deps_enabled; info->deps_built = stats->deps_built; result = lappend(result, info); @@ -274,6 +276,7 @@ compare_scalars_partition(const void *a, const void *b, void *arg) return ApplySortComparator(da, false, db, false, ssup); } + /* initialize multi-dimensional sort */ MultiSortSupport multi_sort_init(int ndims) @@ -354,3 +357,23 @@ multi_sort_compare_dim(int dim, const SortItem *a, const SortItem *b, b->values[dim], b->isnull[dim], &mss->ssup[dim]); } + +int +multi_sort_compare_dims(int start, int end, + const SortItem *a, const SortItem *b, + MultiSortSupport mss) +{ + int dim; + + for (dim = start; dim <= end; dim++) + { + int r = ApplySortComparator(a->values[dim], a->isnull[dim], + b->values[dim], b->isnull[dim], + &mss->ssup[dim]); + + if (r != 0) + return r; + } + + return 0; +} diff --git a/src/backend/utils/mvstats/common.h b/src/backend/utils/mvstats/common.h index d96422d..a019ea6 100644 --- a/src/backend/utils/mvstats/common.h +++ b/src/backend/utils/mvstats/common.h @@ -70,6 +70,9 @@ int multi_sort_compare(const void *a, const void *b, void *arg); int multi_sort_compare_dim(int dim, const SortItem *a, const SortItem *b, MultiSortSupport mss); +int multi_sort_compare_dims(int start, int end, const SortItem *a, + const SortItem *b, MultiSortSupport mss); + /* comparators, used when constructing multivariate stats */ int compare_scalars_simple(const void *a, const void *b, void *arg); int compare_scalars_partition(const void *a, const void *b, void *arg); diff --git a/src/backend/utils/mvstats/dependencies.c b/src/backend/utils/mvstats/dependencies.c index 2a064a0..412dc30 100644 --- a/src/backend/utils/mvstats/dependencies.c +++ b/src/backend/utils/mvstats/dependencies.c @@ -17,293 +17,521 @@ #include "common.h" #include "utils/lsyscache.h" +/* internal state for generator of variations (k-permutations of n elements) */ +typedef struct VariationGeneratorData { + + int k; /* size of the k-permutation */ + int current; /* index of the next variation to return */ + + int nvariations; /* number of variations generated (size of array) */ + int variations[1]; /* array of pre-built variations */ + +} VariationGeneratorData; + +typedef VariationGeneratorData* VariationGenerator; + +/* + * generate all variations (k-permutations of n elements) + */ +static void +generate_variations(VariationGenerator state, + int n, int maxlevel, int level, int *current) +{ + int i, j; + + /* initialize */ + if (level == 0) + { + current = (int*)palloc0(sizeof(int) * (maxlevel+1)); + state->current = 0; + } + + for (i = 0; i < n; i++) + { + /* check if the value is already used current variation */ + bool found = false; + for (j = 0; j < level; j++) + { + if (current[j] == i) + { + found = true; + break; + } + } + + /* already used, so try the next element */ + if (found) + continue; + + /* ok, we can use this element, so store it */ + current[level] = i; + + /* and check if we do have a complete variation of k elements */ + if (level == maxlevel) + { + /* yep, store the variation */ + Assert(state->current < state->nvariations); + memcpy(&state->variations[(state->k * state->current)], current, + sizeof(int) * (maxlevel+1)); + state->current++; + } + else + /* nope, look for additional elements */ + generate_variations(state, n, maxlevel, level+1, current); + } + + if (level == 0) + pfree(current); +} + /* - * Detect functional dependencies between columns. + * initialize the generator of variations, and prebuild the variations * - * TODO This builds a complete set of dependencies, i.e. including transitive - * dependencies - if we identify [A => B] and [B => C], we're likely to - * identify [A => C] too. It might be better to keep only the minimal set - * of dependencies, i.e. prune all the dependencies that we can recreate - * by transivitity. - * - * There are two conceptual ways to do that: - * - * (a) generate all the rules, and then prune the rules that may be - * recteated by combining other dependencies, or - * - * (b) performing the 'is combination of other dependencies' check before - * actually doing the work - * - * The second option has the advantage that we don't really need to perform - * the sort/count. It's not sufficient alone, though, because we may - * discover the dependencies in the wrong order. For example we may find + * This pre-builds all the variations. We could also generate them in + * generator_next(), but this seems simpler. + */ +static VariationGenerator +generator_init(int2vector *attrs, int k) +{ + int i; + int n = attrs->dim1; + int nvariations; + VariationGenerator state; + + Assert((n >= k) && (k > 0)); + + /* compute the total number of variations as n!/(n-k)! */ + nvariations = n; + for (i = 1; i < k; i++) + nvariations *= (n - i); + + /* allocate the generator state as a single chunk of memory */ + state = (VariationGenerator)palloc0( + offsetof(VariationGeneratorData, variations) + + (nvariations * k * sizeof(int))); /* variations */ + + state->nvariations = nvariations; + state->k = k; + + /* now actually pre-generate all the variations */ + generate_variations(state, n, (k-1), 0, NULL); + + /* we expect to generate exactly the right number of variations */ + Assert(state->nvariations == state->current); + + /* reset the index */ + state->current = 0; + + return state; +} + +/* free the generator state */ +static void +generator_free(VariationGenerator state) +{ + /* we've allocated a single chunk, so just free it */ + pfree(state); +} + +/* generate next combination */ +static int* +generator_next(VariationGenerator state, int2vector *attrs) +{ + if (state->current == state->nvariations) + return NULL; + + return &state->variations[state->k * state->current++]; +} + +/* + * check if the dependency is implied by existing dependencies + * + * A dependency is considered implied, if there exists a dependency with the + * same column on the left, and a subset of columns on the right side. So for + * example if we have a dependency + * + * (a,b,c) -> d * - * (a -> b), (a -> c) and then (b -> c) + * then we are looking for these six dependencies * - * None of those dependencies is a combination of the already known ones, - * yet (a -> C) is a combination of (a -> b) and (b -> c). + * (a) -> d + * (b) -> d + * (c) -> d + * (a,b) -> d + * (a,c) -> d + * (b,c) -> d * - * - * FIXME Currently we simply replace NULL values with 0 and then handle is as - * a regular value, but that groups NULL and actual 0 values. That's - * clearly incorrect - we need to handle NULL values as a separate value. + * This does not detect transitive dependencies. For example if we have + * + * (a) -> b + * (b) -> c + * + * then obviously + * + * (a) -> c + * + * but this is not detected. Extending the method to handle transitive cases + * is future work. */ -MVDependencies -build_mv_dependencies(int numrows, HeapTuple *rows, int2vector *attrs, - VacAttrStats **stats) +static bool +dependency_is_implied(MVDependencies dependencies, int k, int *dependency, + int2vector * attrs) { - int i; - int numattrs = attrs->dim1; + bool implied = false; + int i, j, l; + int *tmp; - /* result */ - int ndeps = 0; - MVDependencies dependencies = NULL; - MultiSortSupport mss = multi_sort_init(2); /* 2 dimensions for now */ + if (dependencies == NULL) + return false; - /* TODO Maybe this should be somehow related to the number of - * distinct values in the two columns we're currently analyzing. - * Assuming the distribution is uniform, we can estimate the - * average group size and use it as a threshold. Or something - * like that. Seems better than a static approach. - */ - int min_group_size = 3; + tmp = (int*)palloc0(sizeof(int) * k); + + /* translate the indexes to actual attribute numbers */ + for (i = 0; i < k; i++) + tmp[i] = attrs->values[dependency[i]]; + + /* search for a smaller */ + for (i = 0; i < dependencies->ndeps; i++) + { + bool contained = true; + MVDependency dep = dependencies->deps[i]; + + /* does the last attribute match? */ + if (tmp[k-1] != dep->attributes[dep->nattributes-1]) + continue; /* nope, no need to check this dependency further */ + + /* are the conditions superset of the existing dependency? */ + for (j = 0; j < (dep->nattributes-1); j++) + { + bool found = false; + + for (l = 0; l < (k-1); l++) + { + if (tmp[l] == dep->attributes[j]) + { + found = true; + break; + } + } + + /* we've found an attribute not included in the new dependency */ + if (! found) + { + contained = false; + break; + } + } + + /* we've found an existing dependency, trivially proving the new one */ + if (contained) + { + implied = true; + break; + } + } - /* dimension indexes we'll check for associations [a => b] */ - int dima, dimb; + pfree(tmp); + + return implied; +} + +/* + * validates functional dependency on the data + * + * An actual work horse of detecting functional dependencies. Given a variation + * of k attributes, it checks that the first (k-1) are sufficient to determine + * the last one. + */ +static bool +dependency_is_valid(int numrows, HeapTuple *rows, int k, int * dependency, + VacAttrStats **stats, int2vector *attrs) +{ + int i, j; + int nvalues = numrows * k; /* - * We'll reuse the same array for all the 2-column combinations. - * - * It's possible to sort the sample rows directly, but this seemed - * somehow simples / less error prone. Another option would be to - * allocate the arrays for each SortItem separately, but that'd be - * significant overhead (not just CPU, but especially memory bloat). + * XXX Maybe the threshold should be somehow related to the number of + * distinct values in the combination of columns we're analyzing. + * Assuming the distribution is uniform, we can estimate the average + * group size and use it as a threshold, similarly to what we do for + * MCV lists. */ - SortItem * items = (SortItem*)palloc0(numrows * sizeof(SortItem)); + int min_group_size = 3; + + /* number of groups supporting / contradicting the dependency */ + int n_supporting = 0; + int n_contradicting = 0; + + /* counters valid within a group */ + int group_size = 0; + int n_violations = 0; - Datum *values = (Datum*)palloc0(sizeof(Datum) * numrows * 2); - bool *isnull = (bool*)palloc0(sizeof(bool) * numrows * 2); + int n_supporting_rows = 0; + int n_contradicting_rows = 0; + /* sort info for all attributes columns */ + MultiSortSupport mss = multi_sort_init(k); + + /* data for the sort */ + SortItem *items = (SortItem*)palloc0(numrows * sizeof(SortItem)); + Datum *values = (Datum*)palloc0(sizeof(Datum) * nvalues); + bool *isnull = (bool*)palloc0(sizeof(bool) * nvalues); + + /* fix the pointers to values/isnull */ for (i = 0; i < numrows; i++) { - items[i].values = &values[i * 2]; - items[i].isnull = &isnull[i * 2]; + items[i].values = &values[i * k]; + items[i].isnull = &isnull[i * k]; } - Assert(numattrs >= 2); - /* - * Evaluate all possible combinations of [A => B], using a simple algorithm: + * Verify the dependency (a,b,...)->z, using a rather simple algorithm: + * + * (a) sort the data lexicographically * - * (a) sort the data by [A,B] - * (b) split the data into groups by A (new group whenever a value changes) - * (c) count different values in the B column (again, value changes) + * (b) split the data into groups by first (k-1) columns * - * TODO It should be rather simple to merge [A => B] and [A => C] into - * [A => B,C]. Just keep A constant, collect all the "implied" columns - * and you're done. + * (c) for each group count different values in the last column */ - for (dima = 0; dima < numattrs; dima++) + + /* prepare the sort function for the first dimension, and SortItem array */ + for (i = 0; i < k; i++) { - /* prepare the sort function for the first dimension */ - multi_sort_add_dimension(mss, 0, dima, stats); + multi_sort_add_dimension(mss, i, dependency[i], stats); - for (dimb = 0; dimb < numattrs; dimb++) + /* accumulate all the data for both columns into an array and sort it */ + for (j = 0; j < numrows; j++) { - SortItem current; - - /* number of groups supporting / contradicting the dependency */ - int n_supporting = 0; - int n_contradicting = 0; - - /* counters valid within a group */ - int group_size = 0; - int n_violations = 0; - - int n_supporting_rows = 0; - int n_contradicting_rows = 0; - - /* make sure the columns are different (A => A) */ - if (dima == dimb) - continue; - - /* prepare the sort function for the second dimension */ - multi_sort_add_dimension(mss, 1, dimb, stats); - - /* reset the values and isnull flags */ - memset(values, 0, sizeof(Datum) * numrows * 2); - memset(isnull, 0, sizeof(bool) * numrows * 2); + items[j].values[i] + = heap_getattr(rows[j], attrs->values[dependency[i]], + stats[i]->tupDesc, &items[j].isnull[i]); + } + } - /* accumulate all the data for both columns into an array and sort it */ - for (i = 0; i < numrows; i++) - { - items[i].values[0] - = heap_getattr(rows[i], attrs->values[dima], - stats[dima]->tupDesc, &items[i].isnull[0]); + /* sort the items so that we can detect the groups */ + qsort_arg((void *) items, numrows, sizeof(SortItem), + multi_sort_compare, mss); - items[i].values[1] - = heap_getattr(rows[i], attrs->values[dimb], - stats[dimb]->tupDesc, &items[i].isnull[1]); - } + /* + * Walk through the sorted array, split it into rows according to the first + * (k-1) columns. If there's a single value in the last column, we count + * the group as 'supporting' the functional dependency. Otherwise we count + * it as contradicting. + * + * We also require a group to have a minimum number of rows to be considered + * useful for supporting the dependency. Contradicting groups may be of + * any size, though. + * + * XXX The minimum size requirement makes it impossible to identify case + * when both columns are unique (or nearly unique), and therefore + * trivially functionally dependent. + */ - qsort_arg((void *) items, numrows, sizeof(SortItem), - multi_sort_compare, mss); + /* start with the first row forming a group */ + group_size = 1; + for (i = 1; i < numrows; i++) + { + /* end of the preceding group */ + if (multi_sort_compare_dims(0, (k-2), &items[i-1], &items[i], mss) != 0) + { /* - * Walk through the array, split it into rows according to - * the A value, and count distinct values in the other one. - * If there's a single B value for the whole group, we count - * it as supporting the association, otherwise we count it - * as contradicting. - * - * Furthermore we require a group to have at least a certain - * number of rows to be considered useful for supporting the - * dependency. But when it's contradicting, use it always useful. + * If there is a single are no contradicting rows, count the group + * as supporting, otherwise contradicting. */ - - /* start with values from the first row */ - current = items[0]; - group_size = 1; - - for (i = 1; i < numrows; i++) - { - /* end of the group */ - if (multi_sort_compare_dim(0, &items[i], ¤t, mss) != 0) - { - /* - * If there are no contradicting rows, count it as - * supporting (otherwise contradicting), but only if - * the group is large enough. - * - * The requirement of a minimum group size makes it - * impossible to identify [unique,unique] cases, but - * that's probably a different case. This is more - * about [zip => city] associations etc. - * - * If there are violations, count the group/rows as - * a violation. - * - * It may ne neither, if the group is too small (does - * not contain at least min_group_size rows). - */ - if ((n_violations == 0) && (group_size >= min_group_size)) - { - n_supporting += 1; - n_supporting_rows += group_size; - } - else if (n_violations > 0) - { - n_contradicting += 1; - n_contradicting_rows += group_size; - } - - /* current values start a new group */ - n_violations = 0; - group_size = 0; - } - /* mismatch of a B value is contradicting */ - else if (multi_sort_compare_dim(1, &items[i], ¤t, mss) != 0) - { - n_violations += 1; - } - - current = items[i]; - group_size += 1; - } - - /* handle the last group (just like above) */ if ((n_violations == 0) && (group_size >= min_group_size)) { - n_supporting += 1; + n_supporting += 1; n_supporting_rows += group_size; } - else if (n_violations) + else if (n_violations > 0) { - n_contradicting += 1; + n_contradicting += 1; n_contradicting_rows += group_size; } - /* - * See if the number of rows supporting the association is at least - * 10x the number of rows violating the hypothetical dependency. - * - * TODO This is rather arbitrary limit - I guess it's possible to do - * some math to come up with a better rule (e.g. testing a hypothesis - * 'this is due to randomness'). We can create a contingency table - * from the values and use it for testing. Possibly only when - * there are no contradicting rows? - * - * TODO Also, if (a => b) and (b => a) at the same time, it pretty much - * means there's a 1:1 relation (or one is a 'label'), making the - * conditions rather redundant. Although it's possible that the - * query uses incompatible combination of values. - */ - if (n_supporting_rows > (n_contradicting_rows * 10)) - { - if (dependencies == NULL) - { - dependencies = (MVDependencies)palloc0(sizeof(MVDependenciesData)); - dependencies->magic = MVSTAT_DEPS_MAGIC; - } - else - dependencies = repalloc(dependencies, offsetof(MVDependenciesData, deps) - + sizeof(MVDependency) * (dependencies->ndeps + 1)); + /* current values start a new group */ + n_violations = 0; + group_size = 0; + } + /* first colums match, but the last one does not (so contradicting) */ + else if (multi_sort_compare_dims((k-1), (k-1), &items[i-1], &items[i], mss) != 0) + n_violations += 1; - /* update the */ - dependencies->deps[ndeps] = (MVDependency)palloc0(sizeof(MVDependencyData)); - dependencies->deps[ndeps]->a = attrs->values[dima]; - dependencies->deps[ndeps]->b = attrs->values[dimb]; + group_size += 1; + } - dependencies->ndeps = (++ndeps); - } - } + /* handle the last group (just like above) */ + if ((n_violations == 0) && (group_size >= min_group_size)) + { + n_supporting += 1; + n_supporting_rows += group_size; + } + else if (n_violations) + { + n_contradicting += 1; + n_contradicting_rows += group_size; } pfree(items); pfree(values); pfree(isnull); - pfree(stats); pfree(mss); - return dependencies; + /* + * See if the number of rows supporting the association is at least 10x the + * number of rows violating the hypothetical dependency. + */ + return (n_supporting_rows > (n_contradicting_rows * 10)); } /* - * Store the dependencies into a bytea, so that it can be stored in the - * pg_mv_statistic catalog. + * detects functional dependencies between groups of columns + * + * Generates all possible subsets of columns (variations) and checks if the + * last one is determined by the preceding ones. For example given 3 columns, + * there are 12 variations (6 for variations on 2 columns, 6 for 3 columns): + * + * two columns three columns + * ----------- ------------- + * (a) -> c (a,b) -> c + * (b) -> c (b,a) -> c + * (a) -> b (a,c) -> b + * (c) -> b (c,a) -> b + * (c) -> a (c,b) -> a + * (b) -> a (b,c) -> a + * + * Clearly some of the variations are redundant, as the order of columns on the + * left side does not matter. This is detected in dependency_is_implied, and + * those dependencies are ignored. * - * Currently this only supports simple two-column rules, and stores them - * as a sequence of attnum pairs. In the future, this needs to be made - * more complex to support multiple columns on both sides of the - * implication (using AND on left, OR on right). + * We however do not detect that dependencies are transitively implied. For + * example given dependencies + * + * (a) -> b + * (b) -> c + * + * then + * + * (a) -> c + * + * is trivially implied. However we don't detect that and all three dependencies + * will get included in the resulting set. Eliminating such transitively implied + * dependencies is future work. + */ +MVDependencies +build_mv_dependencies(int numrows, HeapTuple *rows, int2vector *attrs, + VacAttrStats **stats) +{ + int i; + int k; + int numattrs = attrs->dim1; + + /* result */ + MVDependencies dependencies = NULL; + + Assert(numattrs >= 2); + + /* + * We'll try build functional dependencies starting from the smallest ones + * covering jut 2 columns, to the largest ones, covering all columns + * included int the statistics. We start from the smallest ones because + * we want to be able to skip already implied ones. + */ + for (k = 2; k <= numattrs; k++) + { + int *dependency; /* array with k elements */ + + /* prepare a generator of variation */ + VariationGenerator generator = generator_init(attrs, k); + + /* generate all possible variations of k values (out of n) */ + while ((dependency = generator_next(generator, attrs))) + { + MVDependency d; + + /* skip dependencies that are already trivially implied */ + if (dependency_is_implied(dependencies, k, dependency, attrs)) + continue; + + /* also skip dependencies that don't seem to be valid */ + if (! dependency_is_valid(numrows, rows, k, dependency, stats, attrs)) + continue; + + d = (MVDependency)palloc0(offsetof(MVDependencyData, attributes) + + k * sizeof(int)); + + /* copy the dependency, but translate it to actuall attnums */ + d->nattributes = k; + for (i = 0; i < k; i++) + d->attributes[i] = attrs->values[dependency[i]]; + + /* initialize the list of dependencies */ + if (dependencies == NULL) + { + dependencies + = (MVDependencies)palloc0(sizeof(MVDependenciesData)); + + dependencies->magic = MVSTAT_DEPS_MAGIC; + dependencies->type = MVSTAT_DEPS_TYPE_BASIC; + dependencies->ndeps = 0; + } + + dependencies->ndeps++; + dependencies = (MVDependencies)repalloc(dependencies, + offsetof(MVDependenciesData, deps) + + dependencies->ndeps * sizeof(MVDependency)); + + dependencies->deps[dependencies->ndeps-1] = d; + } + + /* we're done with variations of k elements, so free the generator */ + generator_free(generator); + } + + return dependencies; +} + + +/* + * serialize list of dependencies into a bytea */ bytea * serialize_mv_dependencies(MVDependencies dependencies) { int i; + bytea * output; + char *tmp; - /* we need to store ndeps, and each needs 2 * int16 */ + /* we need to store ndeps, with a number of attributes for each one */ Size len = VARHDRSZ + offsetof(MVDependenciesData, deps) - + dependencies->ndeps * (sizeof(int16) * 2); - - bytea * output = (bytea*)palloc0(len); + + sizeof(int) * dependencies->ndeps; - char * tmp = VARDATA(output); + /* and also include space for the actual attribute numbers */ + for (i = 0; i < dependencies->ndeps; i++) + len += (sizeof(int16) * dependencies->deps[i]->nattributes); + output = (bytea*)palloc0(len); SET_VARSIZE(output, len); + tmp = VARDATA(output); + /* first, store the number of dimensions / items */ memcpy(tmp, dependencies, offsetof(MVDependenciesData, deps)); tmp += offsetof(MVDependenciesData, deps); - /* walk through the dependencies and copy both columns into the bytea */ + /* store number of attributes and attribute numbers for each dependency */ for (i = 0; i < dependencies->ndeps; i++) { - memcpy(tmp, &(dependencies->deps[i]->a), sizeof(int16)); - tmp += sizeof(int16); + MVDependency d = dependencies->deps[i]; + + memcpy(tmp, &(d->nattributes), sizeof(int)); + tmp += sizeof(int); - memcpy(tmp, &(dependencies->deps[i]->b), sizeof(int16)); - tmp += sizeof(int16); + memcpy(tmp, d->attributes, sizeof(int16) * d->nattributes); + tmp += sizeof(int16) * d->nattributes; + + Assert(tmp <= ((char*)output + len)); } return output; @@ -338,20 +566,21 @@ deserialize_mv_dependencies(bytea * data) tmp += offsetof(MVDependenciesData, deps); if (dependencies->magic != MVSTAT_DEPS_MAGIC) - { - pfree(dependencies); - elog(WARNING, "not a MV Dependencies (magic number mismatch)"); - return NULL; - } + elog(ERROR, "invalid dependency type %d (expected %dd)", + dependencies->type, MVSTAT_DEPS_MAGIC); + + if (dependencies->type != MVSTAT_DEPS_TYPE_BASIC) + elog(ERROR, "invalid dependency type %d (expected %dd)", + dependencies->type, MVSTAT_DEPS_TYPE_BASIC); Assert(dependencies->ndeps > 0); - /* what bytea size do we expect for those parameters */ + /* what minimum bytea size do we expect for those parameters */ expected_size = offsetof(MVDependenciesData,deps) + - dependencies->ndeps * sizeof(int16) * 2; + dependencies->ndeps * (sizeof(int) + sizeof(int16) * 2); - if (VARSIZE_ANY_EXHDR(data) != expected_size) - elog(ERROR, "invalid dependencies size %ld (expected %ld)", + if (VARSIZE_ANY_EXHDR(data) < expected_size) + elog(ERROR, "invalid dependencies size %ld (expected at least %ld)", VARSIZE_ANY_EXHDR(data), expected_size); /* allocate space for the MCV items */ @@ -360,15 +589,35 @@ deserialize_mv_dependencies(bytea * data) for (i = 0; i < dependencies->ndeps; i++) { - dependencies->deps[i] = (MVDependency)palloc0(sizeof(MVDependencyData)); + int k; + MVDependency d; + + /* number of attributes */ + memcpy(&k, tmp, sizeof(int)); + tmp += sizeof(int); + + /* is the number of attributes valid? */ + Assert((k >= 2) && (k <= MVSTATS_MAX_DIMENSIONS)); - memcpy(&(dependencies->deps[i]->a), tmp, sizeof(int16)); - tmp += sizeof(int16); + /* now that we know the number of attributes, allocate the dependency */ + d = (MVDependency)palloc0(offsetof(MVDependencyData, attributes) + + k * sizeof(int)); - memcpy(&(dependencies->deps[i]->b), tmp, sizeof(int16)); - tmp += sizeof(int16); + d->nattributes = k; + + /* copy attribute numbers */ + memcpy(d->attributes, tmp, sizeof(int16) * d->nattributes); + tmp += sizeof(int16) * d->nattributes; + + dependencies->deps[i] = d; + + /* still within the bytea */ + Assert(tmp <= ((char*)data + VARSIZE_ANY(data))); } + /* we should have consumed the whole bytea exactly */ + Assert(tmp == ((char*)data + VARSIZE_ANY(data))); + return dependencies; } @@ -392,46 +641,70 @@ pg_mv_stats_dependencies_info(PG_FUNCTION_ARGS) PG_RETURN_TEXT_P(cstring_to_text(result)); } -/* print the dependencies - * - * TODO Would be nice if this knew the actual column names (instead of - * the attnums). +/* + * print the dependencies * - * FIXME This is really ugly and does not really check the lengths and - * strcpy/snprintf return values properly. Needs to be fixed. + * TODO Would be nice if this printed column names (instead of just attnums). */ Datum pg_mv_stats_dependencies_show(PG_FUNCTION_ARGS) { - int i = 0; - bytea *data = PG_GETARG_BYTEA_P(0); - char *result = NULL; - int len = 0; + int i, j; + bytea *data = PG_GETARG_BYTEA_P(0); + StringInfoData buf; MVDependencies dependencies = deserialize_mv_dependencies(data); if (dependencies == NULL) PG_RETURN_NULL(); + initStringInfo(&buf); + for (i = 0; i < dependencies->ndeps; i++) { MVDependency dependency = dependencies->deps[i]; - char buffer[128]; - int tmp = snprintf(buffer, 128, "%s%d => %d", - ((i == 0) ? "" : ", "), dependency->a, dependency->b); + if (i > 0) + appendStringInfo(&buf, ", "); - if (tmp < 127) + /* conditions */ + appendStringInfoChar(&buf, '('); + for (j = 0; j < dependency->nattributes-1; j++) { - if (result == NULL) - result = palloc0(len + tmp + 1); - else - result = repalloc(result, len + tmp + 1); + if (j > 0) + appendStringInfoChar(&buf, ','); - strcpy(result + len, buffer); - len += tmp; + appendStringInfo(&buf, "%d", dependency->attributes[j]); } + + /* the implied attribute */ + appendStringInfo(&buf, ") => %d", + dependency->attributes[dependency->nattributes-1]); } - PG_RETURN_TEXT_P(cstring_to_text(result)); + PG_RETURN_TEXT_P(cstring_to_text(buf.data)); +} + +MVDependencies +load_mv_dependencies(Oid mvoid) +{ + bool isnull = false; + Datum deps; + + /* Prepare to scan pg_mv_statistic for entries having indrelid = this rel. */ + HeapTuple htup = SearchSysCache1(MVSTATOID, ObjectIdGetDatum(mvoid)); + +#ifdef USE_ASSERT_CHECKING + Form_pg_mv_statistic mvstat = (Form_pg_mv_statistic) GETSTRUCT(htup); + Assert(mvstat->deps_enabled && mvstat->deps_built); +#endif + + deps = SysCacheGetAttr(MVSTATOID, htup, + Anum_pg_mv_statistic_stadeps, &isnull); + + Assert(!isnull); + + ReleaseSysCache(htup); + + return deserialize_mv_dependencies(DatumGetByteaP(deps)); } diff --git a/src/include/utils/mvstats.h b/src/include/utils/mvstats.h index 7ebd961..c6f45ab 100644 --- a/src/include/utils/mvstats.h +++ b/src/include/utils/mvstats.h @@ -17,22 +17,31 @@ #include "fmgr.h" #include "commands/vacuum.h" +/* + * Degree of how much MCV item / histogram bucket matches a clause. + * This is then considered when computing the selectivity. + */ +#define MVSTATS_MATCH_NONE 0 /* no match at all */ +#define MVSTATS_MATCH_PARTIAL 1 /* partial match */ +#define MVSTATS_MATCH_FULL 2 /* full match */ #define MVSTATS_MAX_DIMENSIONS 8 /* max number of attributes */ -/* An associative rule, tracking [a => b] dependency. - * - * TODO Make this work with multiple columns on both sides. + +/* + * Functional dependencies, tracking column-level relationships (values + * in one column determine values in another one). */ typedef struct MVDependencyData { - int16 a; - int16 b; + int nattributes; /* number of attributes */ + int16 attributes[1]; /* attribute numbers */ } MVDependencyData; typedef MVDependencyData* MVDependency; typedef struct MVDependenciesData { uint32 magic; /* magic constant marker */ + uint32 type; /* type of MV Dependencies (BASIC) */ int32 ndeps; /* number of dependencies */ MVDependency deps[1]; /* XXX why not a pointer? */ } MVDependenciesData; @@ -48,6 +57,8 @@ typedef MVDependenciesData* MVDependencies; * stats specified using flags (or something like that). */ +MVDependencies load_mv_dependencies(Oid mvoid); + bytea * serialize_mv_dependencies(MVDependencies dependencies); /* deserialization of stats (serialization is private to analyze) */ diff --git a/src/test/regress/expected/mv_dependencies.out b/src/test/regress/expected/mv_dependencies.out new file mode 100644 index 0000000..e759997 --- /dev/null +++ b/src/test/regress/expected/mv_dependencies.out @@ -0,0 +1,172 @@ +-- data type passed by value +CREATE TABLE functional_dependencies ( + a INT, + b INT, + c INT +); +-- unknown column +CREATE STATISTICS s1 ON functional_dependencies (unknown_column) WITH (dependencies); +ERROR: column "unknown_column" referenced in statistics does not exist +-- single column +CREATE STATISTICS s1 ON functional_dependencies (a) WITH (dependencies); +ERROR: multivariate stats require 2 or more columns +-- single column, duplicated +CREATE STATISTICS s1 ON functional_dependencies (a,a) WITH (dependencies); +ERROR: duplicate column name in statistics definition +-- two columns, one duplicated +CREATE STATISTICS s1 ON functional_dependencies (a, a, b) WITH (dependencies); +ERROR: duplicate column name in statistics definition +-- unknown option +CREATE STATISTICS s1 ON functional_dependencies (a, b, c) WITH (unknown_option); +ERROR: unrecognized STATISTICS option "unknown_option" +-- correct command +CREATE STATISTICS s1 ON functional_dependencies (a, b, c) WITH (dependencies); +-- random data (no functional dependencies) +INSERT INTO functional_dependencies + SELECT mod(i, 111), mod(i, 123), mod(i, 23) FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | f | +(1 row) + +TRUNCATE functional_dependencies; +-- a => b, a => c, b => c +INSERT INTO functional_dependencies + SELECT i/10, i/100, i/200 FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | t | 1 => 2, 1 => 3, 2 => 3 +(1 row) + +TRUNCATE functional_dependencies; +-- a => b, a => c +INSERT INTO functional_dependencies + SELECT i/10, i/150, i/200 FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | t | 1 => 2, 1 => 3 +(1 row) + +TRUNCATE functional_dependencies; +-- check explain (expect bitmap index scan, not plain index scan) +INSERT INTO functional_dependencies + SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i); +CREATE INDEX fdeps_idx ON functional_dependencies (a, b); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | t | 1 => 2, 1 => 3, 2 => 3 +(1 row) + +EXPLAIN (COSTS off) + SELECT * FROM functional_dependencies WHERE a = 10 AND b = 5; + QUERY PLAN +--------------------------------------------- + Bitmap Heap Scan on functional_dependencies + Recheck Cond: ((a = 10) AND (b = 5)) + -> Bitmap Index Scan on fdeps_idx + Index Cond: ((a = 10) AND (b = 5)) +(4 rows) + +DROP TABLE functional_dependencies; +-- varlena type (text) +CREATE TABLE functional_dependencies ( + a TEXT, + b TEXT, + c TEXT +); +CREATE STATISTICS s2 ON functional_dependencies (a, b, c) WITH (dependencies); +-- random data (no functional dependencies) +INSERT INTO functional_dependencies + SELECT mod(i, 111), mod(i, 123), mod(i, 23) FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | f | +(1 row) + +TRUNCATE functional_dependencies; +-- a => b, a => c, b => c +INSERT INTO functional_dependencies + SELECT i/10, i/100, i/200 FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | t | 1 => 2, 1 => 3, 2 => 3 +(1 row) + +TRUNCATE functional_dependencies; +-- a => b, a => c +INSERT INTO functional_dependencies + SELECT i/10, i/150, i/200 FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | t | 1 => 2, 1 => 3 +(1 row) + +TRUNCATE functional_dependencies; +-- check explain (expect bitmap index scan, not plain index scan) +INSERT INTO functional_dependencies + SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i); +CREATE INDEX fdeps_idx ON functional_dependencies (a, b); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+------------------------------- + t | t | 1 => 2, 1 => 3, 2 => 3 +(1 row) + +EXPLAIN (COSTS off) + SELECT * FROM functional_dependencies WHERE a = '10' AND b = '5'; + QUERY PLAN +------------------------------------------------------------ + Bitmap Heap Scan on functional_dependencies + Recheck Cond: ((a = '10'::text) AND (b = '5'::text)) + -> Bitmap Index Scan on fdeps_idx + Index Cond: ((a = '10'::text) AND (b = '5'::text)) +(4 rows) + +DROP TABLE functional_dependencies; +-- NULL values (mix of int and text columns) +CREATE TABLE functional_dependencies ( + a INT, + b TEXT, + c INT, + d TEXT +); +CREATE STATISTICS s3 ON functional_dependencies (a, b, c, d) WITH (dependencies); +INSERT INTO functional_dependencies + SELECT + mod(i, 100), + (CASE WHEN mod(i, 200) = 0 THEN NULL ELSE mod(i,200) END), + mod(i, 400), + (CASE WHEN mod(i, 300) = 0 THEN NULL ELSE mod(i,600) END) + FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + deps_enabled | deps_built | pg_mv_stats_dependencies_show +--------------+------------+---------------------------------------- + t | t | 2 => 1, 3 => 1, 3 => 2, 4 => 1, 4 => 2 +(1 row) + +DROP TABLE functional_dependencies; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index bec0316..4f2ffb8 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -110,3 +110,6 @@ test: event_trigger # run stats by itself because its delay may be insufficient under heavy load test: stats + +# run tests of multivariate stats +test: mv_dependencies diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 7e9b319..097a04f 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -162,3 +162,4 @@ test: with test: xml test: event_trigger test: stats +test: mv_dependencies diff --git a/src/test/regress/sql/mv_dependencies.sql b/src/test/regress/sql/mv_dependencies.sql new file mode 100644 index 0000000..48dea4d --- /dev/null +++ b/src/test/regress/sql/mv_dependencies.sql @@ -0,0 +1,150 @@ +-- data type passed by value +CREATE TABLE functional_dependencies ( + a INT, + b INT, + c INT +); + +-- unknown column +CREATE STATISTICS s1 ON functional_dependencies (unknown_column) WITH (dependencies); + +-- single column +CREATE STATISTICS s1 ON functional_dependencies (a) WITH (dependencies); + +-- single column, duplicated +CREATE STATISTICS s1 ON functional_dependencies (a,a) WITH (dependencies); + +-- two columns, one duplicated +CREATE STATISTICS s1 ON functional_dependencies (a, a, b) WITH (dependencies); + +-- unknown option +CREATE STATISTICS s1 ON functional_dependencies (a, b, c) WITH (unknown_option); + +-- correct command +CREATE STATISTICS s1 ON functional_dependencies (a, b, c) WITH (dependencies); + +-- random data (no functional dependencies) +INSERT INTO functional_dependencies + SELECT mod(i, 111), mod(i, 123), mod(i, 23) FROM generate_series(1,10000) s(i); + +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +TRUNCATE functional_dependencies; + +-- a => b, a => c, b => c +INSERT INTO functional_dependencies + SELECT i/10, i/100, i/200 FROM generate_series(1,10000) s(i); + +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +TRUNCATE functional_dependencies; + +-- a => b, a => c +INSERT INTO functional_dependencies + SELECT i/10, i/150, i/200 FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +TRUNCATE functional_dependencies; + +-- check explain (expect bitmap index scan, not plain index scan) +INSERT INTO functional_dependencies + SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i); +CREATE INDEX fdeps_idx ON functional_dependencies (a, b); +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +EXPLAIN (COSTS off) + SELECT * FROM functional_dependencies WHERE a = 10 AND b = 5; + +DROP TABLE functional_dependencies; + +-- varlena type (text) +CREATE TABLE functional_dependencies ( + a TEXT, + b TEXT, + c TEXT +); + +CREATE STATISTICS s2 ON functional_dependencies (a, b, c) WITH (dependencies); + +-- random data (no functional dependencies) +INSERT INTO functional_dependencies + SELECT mod(i, 111), mod(i, 123), mod(i, 23) FROM generate_series(1,10000) s(i); + +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +TRUNCATE functional_dependencies; + +-- a => b, a => c, b => c +INSERT INTO functional_dependencies + SELECT i/10, i/100, i/200 FROM generate_series(1,10000) s(i); + +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +TRUNCATE functional_dependencies; + +-- a => b, a => c +INSERT INTO functional_dependencies + SELECT i/10, i/150, i/200 FROM generate_series(1,10000) s(i); +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +TRUNCATE functional_dependencies; + +-- check explain (expect bitmap index scan, not plain index scan) +INSERT INTO functional_dependencies + SELECT i/10000, i/20000, i/40000 FROM generate_series(1,1000000) s(i); +CREATE INDEX fdeps_idx ON functional_dependencies (a, b); +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +EXPLAIN (COSTS off) + SELECT * FROM functional_dependencies WHERE a = '10' AND b = '5'; + +DROP TABLE functional_dependencies; + +-- NULL values (mix of int and text columns) +CREATE TABLE functional_dependencies ( + a INT, + b TEXT, + c INT, + d TEXT +); + +CREATE STATISTICS s3 ON functional_dependencies (a, b, c, d) WITH (dependencies); + +INSERT INTO functional_dependencies + SELECT + mod(i, 100), + (CASE WHEN mod(i, 200) = 0 THEN NULL ELSE mod(i,200) END), + mod(i, 400), + (CASE WHEN mod(i, 300) = 0 THEN NULL ELSE mod(i,600) END) + FROM generate_series(1,10000) s(i); + +ANALYZE functional_dependencies; + +SELECT deps_enabled, deps_built, pg_mv_stats_dependencies_show(stadeps) + FROM pg_mv_statistic WHERE starelid = 'functional_dependencies'::regclass; + +DROP TABLE functional_dependencies; -- 2.5.0